cdxcore 0.1.27__tar.gz → 0.1.29__tar.gz

{cdxcore-0.1.27 → cdxcore-0.1.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdxcore
-Version: 0.1.27
+Version: 0.1.29
 Summary: Basic Python Tools; upgraded cdxbasics
 Author-email: Hans Buehler <github@buehler.london>
 License-Expression: MIT

{cdxcore-0.1.27 → cdxcore-0.1.29}/cdxcore/__init__.py

@@ -4,4 +4,4 @@ Created on June 2022
 @author: hansb
 """
 
-__version__ = "0.1.27"  # auto-updated by setup.py
+__version__ = "0.1.29"  # auto-updated by setup.py

{cdxcore-0.1.27 → cdxcore-0.1.29}/cdxcore/subdir.py

@@ -366,6 +366,7 @@ from collections import OrderedDict
 from collections.abc import Collection, Mapping, Callable, Iterable
 from enum import Enum
 from functools import update_wrapper
+from string import Formatter
         
 import json as json
 import gzip as gzip
@@ -375,7 +376,7 @@ from .err import verify, error, warn, fmt as txtfmt
 from .pretty import PrettyObject
 from .verbose import Context
 from .version import Version, version as version_decorator, VersionError
-from .util import fmt_list, fmt_filename, DEF_FILE_NAME_MAP, plain, is_filename
+from .util import fmt_list, fmt_filename, DEF_FILE_NAME_MAP, plain, is_filename, fmt_dict
 from .uniquehash import unique_hash48, UniqueLabel, NamedUniqueHash, named_unique_filename48_8
 
 """
@@ -1031,8 +1032,9 @@ class SubDir(object):
         """
         Delete all of the current directory if ``tclean`` is ``True``
         """
-        if self._tclean:
+        if getattr(self, "_tclean", False):
             self.delete_everything(keep_directory=False)
+        self._path = None
 
     @staticmethod
     def expand_std_root( name ):
@@ -1078,7 +1080,9 @@ class SubDir(object):
 
     def path_exists(self) -> bool:
         """ Whether the current directory exists """
-        return os.path.exists( self._path[:-1] ) if not self._path is None else False
+        if self._path is None:
+            return False
+        return os.path.exists( self._path[:-1] )
         
     # -- a few basic properties --
 
@@ -2193,8 +2197,11 @@ class SubDir(object):
         ----------
         delete_self: bool
             Whether to delete the directory itself as well, or only its contents.
+            If ``True``, the current object will be left in ``None`` state.
+            
         raise_on_error: bool
             ``False`` for silent failure
+            
         ext : str | None, default ``None``
             Extension for keys, or ``None`` for the directory's default.
             Use ``""`` to match all files regardless of extension.
@@ -2228,8 +2235,9 @@ class SubDir(object):
         Deletes the entire sub directory will all contents.
         
         *WARNING:* deletes *all* files and sub-directories, not just those with the present extension.
-        If ``keep_directory`` is ``False``, the directory referred to by this object will also be deleted.
-        In this case, ``self`` will be set to ``None``.
+        If ``keep_directory`` is ``False``, then the directory referred to by this object will also be deleted.
+
+        In this case, ``self`` will be set to ``None`` state.
         """
         if self._path is None:
             return
@@ -2801,15 +2809,16 @@ class SubDir(object):
     # caching
     # -------
 
-    def cache( self,  version             : str = None , *,
-                      dependencies        : list = None, 
-                      label               : Callable = None,
-                      uid                 : Callable = None,
-                      name                : str = None, 
-                      exclude_args        : list[str] = None,
-                      include_args        : list[str] = None,
-                      exclude_arg_types   : list[type] = None,
-                      version_auto_class  : bool = True):
+    def cache( self,  version              : str|None = None , *,
+                      dependencies         : list|None = None, 
+                      label                : Callable|None = None,
+                      uid                  : Callable|None = None,
+                      name                 : str|None = None, 
+                      exclude_args         : list[str]|None = None,
+                      include_args         : list[str]|None = None,
+                      exclude_arg_types    :  list[type]|None = None,
+                      version_auto_class   : bool = True,
+                      name_of_func_name_arg: str = "func_name"):
         """
         Advanced versioned caching for callables.
         
@@ -3158,7 +3167,7 @@ class SubDir(object):
                     return self.x*y
             
             a = A(x=1)
-            f = cache.cache("0.1", id=lambda self, y : f"a.f({y})")(a.f)  # <- decorate bound 'f'.
+            f = cache.cache("0.1", uid=lambda self, y : f"a.f({y})")(a.f)  # <- decorate bound 'f'.
             r = c(y=2)
 
         In this case the function ``f`` is bound to ``a``. The object is added as ``self`` to the function
@@ -3178,9 +3187,13 @@ class SubDir(object):
            :dec:`cdxcore.subdir.SubDir.cache`
            to provide version information at class level. Only version information are provided here.
            
+           You can use :dec:`cdxcore.subdir.SubDir.cache_class` as an alias.
+           
         2) Secondly, decorate ``__init__``. You do not need to specify a version
            for ``__init__`` as its version usually coincides with the version of the class. At ``__init__``
            you define how unique IDs are generated from the parameters passed to object construction.
+           
+           You can use :dec:`cdxcore.subdir.SubDir.cache_init` as an alias.
 
         Simple example:
             
@@ -3190,10 +3203,10 @@ class SubDir(object):
             cache   = SubDir("!/.cache")
             cache.delete_all_content()   # for illustration
             
-            @cache.cache("0.1")
+            @cache.cache_class("0.1")
             class A(object):
                 
-                @cache.cache(exclude_args=['debug'])
+                @cache.cache_init(exclude_args=['debug'])
                 def __init__(self, x, debug):
                     if debug:
                         print("__init__",x)
@@ -3216,10 +3229,10 @@ class SubDir(object):
         
         .. code-block:: python
         
-            @cache.cache("0.1")
+            @cache.cache_class("0.1")
             class A(object):
                 
-                @cache.cache("0.1", id=lambda x, debug: f"A.__init__(x={x})")  # <-- 'self' is not passed to the lambda function; no need to add **_
+                @cache.cache_init(uid=lambda x, debug: f"A.__init__(x={x})")  # <-- 'self' is not passed to the lambda function; no need to add **_
                 def __init__(self, x, debug):
                     if debug:
                         print("__init__",x)
@@ -3227,77 +3240,162 @@ class SubDir(object):
 
         Decorating classes with ``__slots__`` does not yet work.
                                     
-        See also
-        ^^^^^^^^
+        Managing Caching Accross a Project
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-        For project-wide use it is usually inconvenient to control caching at the level of a 
+        For project-wide use it is usually convenient to control caching at the level of a 
         project-wide cache root directory.
-        See :class:`cdxcore.subdir.VersionedCacheRoot` for a thin convenience wrapper around a :class:`cdxcore.subdir.SubDir`
+        The classs :class:`cdxcore.subdir.VersionedCacheRoot` is a thin convenience wrapper around a :class:`cdxcore.subdir.SubDir`
         with a :class:`cdxcore.subdir.CacheController`.
         
+        The idea is to have a central file, ``cache.py`` which contains the central root for caching.
+        We recommend using an environment variable to be able to control the location of this directory
+        out side the code. Here is an example with an environment variable ``PROJECT_CACHE_DIR``::
+            
+            # file cache.py
+            
+            from cdxcore.subdir import VersionedCacheRoot
+            import os as os
+                        
+            cache_root = VersionedCacheRoot(
+                               os.getenv("PROJECT_CACHE_DIR", "!/.cache")
+                               )
+
+        In a particular project file, say ``pipeline.py`` create a file-local cache directory
+        and use it::
+            
+            # file pipeline.py
+            
+            from cache import cache_root
+            
+            cache_dir = cache_root("pipeline")
+            
+            @cache_dir.cache("0.1")
+            def f(x):
+                return x+2
+            
+            @cache_dir.cache("0.1", dependencies=[f])
+            def g(x)
+                return f(x)**2
+            
+            # ...
+            
+        In case you have issues with caching you can use the central root directory to turn on tracing 
+        accross your project:
+
+        .. code-block:: python
+           :emphasize-lines: 4
+
+            from cdxcore.verbose import Context
+            cache_root = VersionedCacheRoot(
+                               os.getenv("PROJECT_CACHE_DIR", "!/.cache"),
+                               debug_verbose=Context.all    # turn full traing on
+                            )
+
         Parameters
         ----------
-        version : str, optional
+        version : str | None, default ``None``
             Version of the function.
             
-            * If ``None`` then ``F`` must be decorated with :dec:`cdxcore.version.version`.
-            * If set, the function ``F`` is first decorated with :dec:`cdxcore.version.version`.
+            * If ``None`` then a common `F`` must be decorated manually with :dec:`cdxcore.version.version`.
+            * If set, the function ``F`` is automatically first decorated with :dec:`cdxcore.version.version` for you.
             
-        dependencies : list[type], optional
+        dependencies : list[type] | None, default ``None``
             A list of version dependencies, either by reference or by name.
             See :dec:`cdxcore.version.version` for details on name lookup if strings are used.
         
-        label : str | Callable
+        label : str | Callable | None, default ``None``
             Specify a human-readable label for the function call given its parameters.
+            
             This label is used to generate the cache file name, and is also printed in when tracing
             hashing operations. Labels are not assumed to be unique, hence a unique hash of
             the label and the parameters to this function will be appended to generate
             the actual cache file name.
             
-            Use ``uid`` instead if ``label`` represents valid unique filenames.
-            
+            Use ``uid`` instead if ``label`` represents valid unique filenames. You cannot specify both ``uid`` and ``label``.
+            If neither ``uid`` and ``label`` are present, ``name`` will be used.
             
             **Usage:**
             
+            * If ``label`` is a ``Callable`` then ``label( func_name=name, **parameters )`` will be called
+              to generate the actual label.
+              
+              The parameter ``func_name`` refers to the qualified
+              name of the function. Its value can be overwitten by ``name``, while the parameter name itself
+              can be overwritten using ``name_of_func_name_arg``, see below.
+
             * If ``label`` is a plain string without ``{}`` formatting: use this string as-is.
             
-            * If ``label`` is a string with ``{}`` formatting, then ``label.format( name=name, **parameters )``
-              will be used to generate the actual label.
+            * If ``label`` is a string with ``{}`` formatting, then ``label.format( func_name=name, **parameters )``
+              will be used to generate the actual label. 
+              
+              The parameter ``func_name`` refers to the qualified
+              name of the function. Its value can be overwitten by ``name``, while the parameter name itself
+              can be overwritten using ``name_of_func_name_arg``, see below.
             
-            * If ``label`` is a ``Callable`` then ``label( name=name, **parameters )`` will be called
-              to generate the actual label.
-
             See above for examples.
             
             ``label`` cannot be used alongside ``uid``.
             
-        uid : str | Callable
+        uid : str | Callable | None, default ``None``
             Alternative to ``label`` which is assumed to generate a unique cache file name. It has the same
-            semantics as ``label``. When used, parameters to the decorated function are not hashed.
+            semantics as ``label``. When used, parameters to the decorated function are not hashed
+            as the ``uid`` is assumed to be already unique. The string must be a valid file name
             
-            ``uid`` be used alongside ``label``.
+            Use ``label`` if the id is not unique. You cannot specify both ``uid`` and ``label``.
+            If neither ``uid`` and ``label`` are present, ``name`` will be used (as non-unique ``label``).
         
-        name : str, optional
-            Name of this function which is used either on its own if neither ``label`` not ``uid`` are used.
-            If either of them is used, ``name`` is passed as a parameter to either the callable or the
-            formatting operator.
+        name : str | None, default ``None``
+            Name of this function which is used either on its own if neither ``label`` not ``uid`` are used,
+            or which passed as a parameter ``func_name`` to either the callable or the
+            formatting operator. See above for more details.
             
             If ``name`` is not specified it defaults to ``__qualname__`` expanded
             by the module name the function is defined in.
         
-        include_args : list[str]
+        include_args : list[str] | None, default ``None``
             List of arguments to include in generating an unqiue ID, or ``None`` for all.
         
-        exclude_args : list[str]:
-            List of arguments to exclude from generating an unique ID.
+        exclude_args : list[str] | None, default ``None``
+            List of arguments to exclude from generating an unique ID. Examples of such non-functional arguments
+            are workflow controls (debugging) and i/o elements.
             
-        exclude_arg_types : list[type]
-            List of parameter types to exclude from generating an unique ID.
+        exclude_arg_types : list[type] | None, default ``None``
+            List of parameter types to exclude from generating an unique ID. Examples of such non-functional arguments
+            are workflow controls (debugging) and i/o elements.
             
-        version_auto_class : bool
+        version_auto_class : bool, default ``True``
             Whether to automaticallty add version dependencies on base classes or, for member functions, on containing
             classes. This is the ``auto_class`` parameter for :dec:`cdxcore.version.version`.
             
+        name_of_func_name_arg : str, default ``"func_name"``
+            When formatting ``label`` or ``uid``, by default ``"func_name"`` is used to refer to the current
+            function name. If there is already a parameter ``func_name`` for the function, an error will be raised.
+            Use this flag to change the parameter name. Example::
+
+            .. code-block:: python
+
+                from cdxcore.subdir import SubDir
+                cache = SubDir("?/temp")
+
+                @cache.cache("0.1")
+                def f( func_name, x ):
+                    pass
+                
+                f("test", 1)
+                
+            Generates a :class:`RuntimeError` ``f@__main__: 'func_name' is a reserved keyword
+            and used as formatting parameter
+            name for the function name. Found it also in the function parameter list. Use 'name_of_name_arg' to change the internal parameter name used.``.
+
+            Instead, use:            
+            
+            .. code-block:: python
+
+                @cache.cache("0.1", label=lambda new_func_name, func_name, x : f"{new_func_name}(): {func_name} {x}", name_of_func_name_arg="new_func_name")
+                def f( func_name, x ):
+                    pass
+
         Returns
         -------
         Decorated F: Callable
@@ -3326,12 +3424,39 @@ class SubDir(object):
                 
             The decorated ``F()`` has additional function parameters, namely:
                 
-            * ``override_cache_mode`` : allows to override caching mode temporarily, in particular you can set it to ``"off"``.
-            * ``track_cached_files`` : allows passing a :class:`cdxcore.subdir.CacheTracker`
+            * ``override_cache_mode`` : ``CacheMode`` | None, default ``None``
+            
+              Allows overriding the ``CacheMode`` temporarily, in particular you can set it to ``"off"``.
+               
+            * ``track_cached_files`` : :class:`cdxcore.subdir.CacheTracker` | None, default ``None``
+            
+              Allows passing a :class:`cdxcore.subdir.CacheTracker`
               object to keep track of all
               files used (loaded from or saved to). 
               The function :meth:`cdxcore.subdir.CacheTracker.delete_cache_files` can be used
               to delete all files involved in caching.
+
+            * ``return_cache_uid`` : bool, default ``False``
+            
+              If ``True``, then the decorated function will return a tuple ``uid, result``
+              where ``uid`` is the unique filename generated for this function call,
+              and where ``result`` is the actual result from the function, cached or not.
+              
+              Usage::
+                  
+                  from cdxcore.subdir import SubDir
+                  cache_dir = SubDir("!/.cache")
+                  
+                  @cache_dir.cache()
+                  def f(x, y):
+                      return x*y
+                  
+                  uid, xy = f( x=1, y=2, return_cache_uid=True )
+                  
+              This pattern is thread-safe when compared to using::
+              
+                  xy = f( x=1, y=2 )
+                  uid = f.cache_info.filename
         """
         return CacheCallable(subdir = self,
                              version = version,
@@ -3342,7 +3467,8 @@ class SubDir(object):
                              exclude_args = exclude_args,
                              include_args = include_args,
                              exclude_arg_types = exclude_arg_types,
-                             version_auto_class = version_auto_class )
+                             version_auto_class = version_auto_class,
+                             name_of_func_name_arg = name_of_func_name_arg)
 
     def cache_class( self, 
                      version             : str = None , *,
@@ -3361,17 +3487,41 @@ class SubDir(object):
             @cache.cache_class("0.1")
             class A(object):
                 
-                @cache.cache(exclude_args=['debug'])
+                @cache.cache_init(exclude_args=['debug'])
                 def __init__(self, x, debug):
                     if debug:
                         print("__init__",x)
                     self.x = x
 
         """
-        return self.cache( name=name,
-                           version=version,
-                           dependencies=dependencies,
-                           version_auto_class=version_auto_class)        
+        return self.cache( name=name, version=version, dependencies=dependencies, version_auto_class=version_auto_class)        
+
+    def cache_init(  self, 
+                     label               : Callable = None,
+                     uid                 : Callable = None,
+                     exclude_args        : list[str] = None,
+                     include_args        : list[str] = None,
+                     exclude_arg_types   : list[type] = None,
+                     ):
+        """
+        Short-cut for :dec:`cdxcore.subdir.SubDir.cache` applied to decorating ``__init__``
+        with a reduced number of available parameters.
+        
+        Example::
+
+            cache   = SubDir("!/.cache")
+            
+            @cache.cache_class("0.1")
+            class A(object):
+                
+                @cache.cache_init(exclude_args=['debug'])
+                def __init__(self, x, debug):
+                    if debug:
+                        print("__init__",x)
+                    self.x = x
+
+        """
+        return self.cache( label=label, uid=uid, exclude_args=exclude_args, include_args=include_args, exclude_arg_types=exclude_arg_types )        
 
 # ========================================================================
 # Caching, convenience
@@ -3444,6 +3594,9 @@ def VersionedCacheRoot( directory          : str, *,
         
         * ``hash_length``: length used for hashes, see :class:`cdxcore.uniquehash.UniqueHash`.
         
+        * ``debug_verbose`` set to ``Context.all`` after importing ``from cdxcore.verbose import Context``
+          will turn on tracing all caching operations.
+        
     Returns
     -------
     Root : :class:`cdxcore.subdir.SubDir`
@@ -3527,7 +3680,7 @@ def _ensure_has_version( F,
                               dependencies=dependencies,
                               auto_class=auto_class)(F)
 
-def _qualified_name( F, name ):
+def _qualified_name( F, name = None ):
     """
     Return qualified name including module name, robustly
     """
@@ -3546,6 +3699,36 @@ def _qualified_name( F, name ):
             warn( f"Cannot determine module name for '{name}' of {type(F)}" )
     return name
 
+def _expected_str_fmt_args(fmt: str):
+    """
+    Inspect a format string and report what arguments it expects.
+    Returns:
+      - auto_positional: count of automatic '{}' fields
+      - positional_indices: explicit numeric field indices used (e.g., {0}, {2})
+      - keywords: named fields used (e.g., {user}, {price})
+    """
+    f = Formatter()
+    pos = set()
+    auto = 0
+    kws = set()
+
+    for literal, field, spec, conv in f.parse(fmt):
+        if field is None:
+            continue
+        # Keep only the first identifier before attribute/index access
+        head = field.split('.')[0].split('[')[0]
+        if head == "":               # '{}' → automatic positional
+            auto += 1
+        elif head.isdigit():         # '{0}', '{2}' → explicit positional
+            pos.add(int(head))
+        else:                        # '{name}' → keyword
+            kws.add(head)
+
+    return PrettyObject( positional=auto,
+                         posindices=pos,
+                         keywords=kws
+                       )
+
 class CacheCallable(object):
     """
     Wrapper for a cached function.
@@ -3554,17 +3737,17 @@ class CacheCallable(object):
     """
     
     def __init__(self, 
-                    subdir              : SubDir, *,
-                    version             : str = None,
-                    dependencies        : list,
-                    label               : Callable = None,
-                    uid                 : Callable = None,
-                    name                : str = None,
-                    exclude_args        : set[str] = None,
-                    include_args        : set[str] = None,
-                    exclude_arg_types   : set[type] = None,
-                    version_auto_class  : bool = True,
-                    name_of_name_arg    : str = "name"):
+                    subdir               : SubDir, *,
+                    version              : str = None,
+                    dependencies         : list,
+                    label                : Callable = None,
+                    uid                  : Callable = None,
+                    name                 : str = None,
+                    exclude_args         : set[str] = None,
+                    include_args         : set[str] = None,
+                    exclude_arg_types    : set[type] = None,
+                    version_auto_class   : bool = True,
+                    name_of_func_name_arg: str = "name"):
         """
         Utility class for :dec:`cdxcore.subdir.SubDir.cache`.
         
@@ -3573,18 +3756,38 @@ class CacheCallable(object):
         if not label is None and not uid is None:
             error("Cannot specify both 'label' and 'uid'.")
         
-        self._subdir              = SubDir(subdir)
-        self._version             = str(version) if not version is None else None
-        self._dependencies        = list(dependencies) if not dependencies is None else None
-        self._label               = label
-        self._uid                 = uid
-        self._name                = str(name) if not name is None else None
-        self._exclude_args        = set(exclude_args) if not exclude_args is None and len(exclude_args) > 0 else None
-        self._include_args        = set(include_args) if not include_args is None and len(include_args) > 0 else None
-        self._exclude_arg_types   = set(exclude_arg_types) if not exclude_arg_types is None and len(exclude_arg_types) > 0 else None
-        self._version_auto_class  = bool(version_auto_class)
-        self._name_of_name_arg    = str(name_of_name_arg)
+        self._subdir                = SubDir(subdir)
+        self._input_version         = str(version) if not version is None else None
+        self._dependencies          = list(dependencies) if not dependencies is None else None
+        self._label                 = label
+        self._uid                   = uid
+        self._name                  = str(name) if not name is None else None
+        self._exclude_args          = set(exclude_args) if not exclude_args is None and len(exclude_args) > 0 else None
+        self._include_args          = set(include_args) if not include_args is None and len(include_args) > 0 else None
+        self._exclude_arg_types     = set(exclude_arg_types) if not exclude_arg_types is None and len(exclude_arg_types) > 0 else None
+        self._version_auto_class    = bool(version_auto_class)
+        self._name_of_func_name_arg = str(name_of_func_name_arg)
+        self._uid_label_params      = None
         
+        if not self.uid_or_label is None:
+            F     = self.uid_or_label
+            which = "'uid'" if not uid is None else "'label'"
+            if isinstance( F, str ):
+                r = _expected_str_fmt_args( F )
+                if len(r.positional) + len(r.posindices) > 0:
+                    raise ValueError("f{which} '{F}' cannot have positional arguments (empty brackets {} or brackets with integer position {1}). Use only named arguments.")
+                self._uid_label_params = list(r.keywords)
+                del r
+            else:
+                if not inspect.isfunction(F):
+                    if not callable(F):
+                        raise ValueError(f"{which} '{_qualified_name(F)}' is not callable")
+                    F = F.__call__
+                    assert inspect.isfunction(F), ("Internal error - function expected")
+                self._uid_label_params = list( inspect.signature(F).parameters )
+            del F, which
+            self._uid_label_params = self._uid_label_params if len(self._uid_label_params) > 0 else None
+
     @property
     def uid_or_label(self) -> Callable:
         """ ID or label """
@@ -3617,6 +3820,10 @@ class CacheCallable(object):
     def global_exclude_arg_types(self) -> list[type]:
         """ Returns ``exclude_arg_types`` of the underlying :class:`cdxcore.subdir.CacheController` """ 
         return self.cache_controller.exclude_arg_types
+    @property
+    def uid_label_params(self) -> list:
+        """ Returns the ``set`` of parameters the ``uid`` or ``label`` function expects """
+        return self._uid_label_params 
     
     def __call__(self, F : Callable):
         """
@@ -3668,7 +3875,7 @@ class CacheCallable(object):
         # apply version
         # this also ensures that __init__ picks up a version dependency on the class itse
         # (as we forceed 'auto_class' to be true)
-        C = _ensure_has_version( C, version=self._version,
+        C = _ensure_has_version( C, version=self._input_version,
                                     dependencies=self._dependencies,
                                     auto_class=self._version_auto_class)
             
@@ -3689,7 +3896,7 @@ class CacheCallable(object):
         # Cannot currently decorate classes.
 
         
-        is_method = inspect.ismethod(F)
+        is_method = inspect.ismethod(F) # for *bound* methods
         if is_method:
             assert not getattr(F, "__self__", None) is None, ("Method type must have __self__...?", F.__qualname__ )
         elif not inspect.isfunction(F):
@@ -3752,11 +3959,12 @@ class CacheCallable(object):
         # -------
         # Decorate now or pick up existing @version
 
-        F = _ensure_has_version( F, version=self._version,
+        F = _ensure_has_version( F, version=self._input_version,
                                     dependencies=self._dependencies,
                                     auto_class=self._version_auto_class,
                                     allow_default=is_new )
-        
+        version = F.version.unique_id64
+
         # name
         # ----
         
@@ -3768,8 +3976,9 @@ class CacheCallable(object):
         exclude_types = ( self._exclude_arg_types if not self._exclude_arg_types is None else set() )\
                       | ( self.global_exclude_arg_types if not self.global_exclude_arg_types is None else set())
 
-        def execute( *args, override_cache_mode : CacheMode = None, 
-                            track_cached_files  : CacheTracker = None,
+        def execute( *args, override_cache_mode : CacheMode|None = None, 
+                            track_cached_files  : CacheTracker|None = None,
+                            return_cache_uid    : bool = False,
                             **kwargs ):     
             """
             Cached execution of the wrapped function
@@ -3785,38 +3994,46 @@ class CacheCallable(object):
             
             uid_or_label = self.uid_or_label
             filename     = None
-            if isinstance(uid_or_label, str) and self.unique:
-                # if 'id' does not contain formatting codes,
-                # and the result is 'unique' then do not bother collecting
-                # function arguments
-                try:
-                    filename = uid_or_label.format()   # throws a KeyError if 'id' contains formatting information
-                except KeyError:
-                    pass
+            if self.unique and self._uid_label_params is None:                    
+                # the string or function do not require any parameters, and is unique
+                assert not uid_or_label is None
+                filename  = uid_or_label if isinstance( uid_or_label, str ) else uid_or_label()
 
-            if not filename is None:
-                # generate name with the unique string provided by the user
                 if not is_filename(filename):
                     raise ValueError(f"The unique filename '{filename}' computed for '{name}' contains invalid characters for filename. When using `uid` make sure that "+\
-                                     "the returned ID is a valid filename (and unique)")
+                                     "the returned ID is a valid filename (and is unique)")
                 label     = filename
-                filename  = self.uniqueFileName( filename )
                 arguments = None
-                
+
             else:
+                # need the list of parameters to compute a hash and/or a label
+                which     = 'uid' if not self._uid is None else 'label'                 
+                
                 # get dictionary of named arguments
                 arguments = execute.cache_info.signature.bind(*args,**kwargs)
                 arguments.apply_defaults()
                 arguments = arguments.arguments # ordered dict
 
+                # delete 'cls' from argument list for class functions
                 if is_new:
-                    # delete 'cls' from argument list
                     assert len(arguments) >= 1, ("*** Internal error", F.__qualname__, is_new, arguments)
                     del arguments[list(arguments)[0]]
-                argus     = set(arguments)
+
+                # add 'self' for methods                
+                if is_method:
+                    # add __self__ to the beginning of all arguments
+                    full_arguments = OrderedDict()
+                    if is_method:
+                        if 'self' in set(arguments):
+                            raise RuntimeError(f"'self' found in bound method '{name}' argument list {fmt_dict(execute.cache_info.signature.bind(*args,**kwargs).arguments)}.")
+                        full_arguments['self'] = F.__self__
+                    full_arguments |= arguments
+                    arguments = full_arguments
+                    del full_arguments
 
                 # filter dictionary
                 if not self._exclude_args is None or not self._include_args is None:
+                    argus     = set(arguments)
                     excl = set(self._exclude_args) if not self._exclude_args is None else set()
                     if not self._exclude_args is None: 
                         if self._exclude_args > argus:
@@ -3830,7 +4047,7 @@ class CacheCallable(object):
                     for arg in excl:
                         if arg in arguments:
                             del arguments[arg]
-                    del excl
+                    del excl, argus
 
                 if len(exclude_types) > 0:
                     excl = []
@@ -3841,41 +4058,45 @@ class CacheCallable(object):
                         if arg in arguments:
                             del arguments[arg]
                                 
-                # did the user provide a label or unique ID?
                 if uid_or_label is None:
+                    # no label or unique ID
+                    assert not self.unique
                     uid_or_label = name
                     
+                elif self._uid_label_params is None:
+                    # label function or string does not need any parameters
+                    assert not self.unique
+                    uid_or_label = uid_or_label if isinstance( uid_or_label, str ) else uid_or_label()
+                    
                 else:
-                    if self._name_of_name_arg in arguments:
-                        error(f"{name}: '{self._name_of_name_arg}' is a reserved keyword and used as parameter name for the function name. Found it also in the function parameter list. Use 'name_of_name_arg' to change the internal parameter name used.")
-
-                    # add standard arguments
-                    full_arguments = OrderedDict()
-                    if is_method:
-                        assert not 'self' in set(arguments), ("__self__ found in bound method argument list...?", F.__qualname__, execute.cache_info.signature.bind(*args,**kwargs).arguments )
-                        full_arguments['self'] = F.__self__                    
-                    full_arguments[self._name_of_name_arg] = name
-                    for k,v in arguments.items():
-                        full_arguments[k] = v
-                    arguments = full_arguments
-                    del full_arguments, k, v
+                    # function or format string required parameters
+                    # add parameters in order of label/uid parameters
+                    assert not self._uid_label_params is None
+                    
+                    fmt_arguments = {}
+                    for k in self._uid_label_params:
+                        if k == self._name_of_func_name_arg:
+                            if self._name_of_func_name_arg in arguments:
+                                error(f"{name}: '{self._name_of_func_name_arg}' is a reserved keyword for '{which}' which refers to the current function name. "
+                                      "Found it also in the function parameter list. Use 'name_of_func_name_arg' to change the internal parameter name used.")
+                            fmt_arguments[k] = name
+                        else:
+                            if not k in arguments:
+                                args_ = [ f"'{_}'" for _ in arguments ]
+                                raise ValueError(f"Error while generating '{which}' for '{name}': formatting function expected a parameter '{k}' which is not present "+\
+                                                 f"in the list of parameters passed to '{name}': {fmt_list(args_)}.")
+                            fmt_arguments[k] = arguments[k]
 
                     # call format or function                    
                     if isinstance( uid_or_label, str ):
-                        try:
-                            uid_or_label = str.format( uid_or_label, **arguments )
-                        except KeyError as e:
-                            raise KeyError(e, f"Error while generating id for '{name}' using format string '{uid_or_label}': {e}. Available arguments: {list(arguments)}")
-
+                        uid_or_label = str.format( uid_or_label, **fmt_arguments )
                     else:
-                        which = 'uid' if not self._uid is None else 'label'
                         try:
-                            uid_or_label = uid_or_label(**arguments)
-                        except TypeError as e:
-                            raise TypeError(e, f"Error while generating '{which}' for '{name}' using a function: {e}. Available arguments: {list(arguments)}")
+                            uid_or_label = uid_or_label(**fmt_arguments)
                         except Exception as e:
-                            raise type(e)(f"Error while generating '{which}' for '{name}': attempt to call '{which}' of type {type(uid_or_label)} failed: {e}")
-                        assert isinstance(uid_or_label, str), ("Error:", which, "callable must return a string. Found",type(uid_or_label))
+                            raise type(e)(f"Error while generating '{which}' for '{name}': attempt to call '{which}' of callable type {type(uid_or_label)} failed: {e}")
+                        if not isinstance(uid_or_label, str):
+                            raise ValueError("Error calling callable '{which}' for '{name}': callable must return a string. Found {type(uid_or_label))}")
 
                 if self.unique:
                     if not is_filename(uid_or_label):
@@ -3891,7 +4112,6 @@ class CacheCallable(object):
             # determine version, cache mode
             # ------------------
 
-            version_   = self._version if not self._version is None else F.version.unique_id64
             cache_mode = CacheMode(override_cache_mode) if not override_cache_mode is None else self.cache_mode
             del override_cache_mode
 
@@ -3899,8 +4119,8 @@ class CacheCallable(object):
             # -------------------------
 
             execute.cache_info.label    = str(label) if not label is None else None
-            execute.cache_info.filename = filename
-            execute.cache_info.version  = version_
+            execute.cache_info.filename = filename # that is the unique ID for this call
+            execute.cache_info.version  = version
             
             if self.cache_controller.keep_last_arguments:
                 info_arguments = OrderedDict()
@@ -3919,11 +4139,11 @@ class CacheCallable(object):
                     pass
                 tag = Tag()
                 if not is_new:
-                    r = self._subdir.read( filename, tag, version=version_ )
+                    r = self._subdir.read( filename, tag, version=version )
                 else:
                     try:
                         execute.__new_during_read = True
-                        r = self._subdir.read( filename, tag, version=version_ )
+                        r = self._subdir.read( filename, tag, version=version )
                     finally:
                         execute.__new_during_read = False
                         
@@ -3932,11 +4152,13 @@ class CacheCallable(object):
                         track_cached_files += self._fullFileName(filename)
                     execute.cache_info.last_cached = True 
                     if not debug_verbose is None:
-                        debug_verbose.write(f"cache({name}): read '{label}' version 'version {version_}' from cache '{self._subdir.full_file_name(filename)}'.")
+                        debug_verbose.write(f"cache({name}): read '{label}' version 'version {version}' from cache '{self._subdir.full_file_name(filename)}'.")
                     if is_new:
                         assert r.__magic_cache_call_init__ is None, ("**** Internal error. __init__ should reset __magic_cache_call_init__", F.__qualname__, label)
                         r.__magic_cache_call_init__ = False # since we called __new__, __init__ will be called next
 
+                    if return_cache_uid:
+                        return filename, r
                     return r
             
             r = F(*args, **kwargs)
@@ -3951,7 +4173,7 @@ class CacheCallable(object):
                 assert r.__magic_cache_call_init__ is None, ("**** Internal error. __init__ should reset __magic_cache_call_init__")
             
             if cache_mode.write:
-                self._subdir.write(filename,r,version=version_)      
+                self._subdir.write(filename,r,version=version)      
                 if not track_cached_files is None:
                     track_cached_files += self._subdir.full_file_name(filename)
             execute.cache_info.last_cached = False
@@ -3963,9 +4185,12 @@ class CacheCallable(object):
             
             if not debug_verbose is None:
                 if cache_mode.write:
-                    debug_verbose.write(f"cache({name}): called '{label}' version 'version {version_}' and wrote result into '{self._subdir.full_file_name(filename)}'.")
+                    debug_verbose.write(f"cache({name}): called '{label}' version 'version {version}' and wrote result into '{self._subdir.full_file_name(filename)}'.")
                 else:
-                    debug_verbose.write(f"cache({name}): called '{label}' version 'version {version_}' but did *not* write into '{self._subdir.full_file_name(filename)}'.")
+                    debug_verbose.write(f"cache({name}): called '{label}' version 'version {version}' but did *not* write into '{self._subdir.full_file_name(filename)}'.")
+
+            if return_cache_uid:
+                return filename, r
             return r
 
         update_wrapper( wrapper=execute, wrapped=F )

{cdxcore-0.1.27 → cdxcore-0.1.29}/cdxcore/uniquehash.py

@@ -266,11 +266,10 @@ class UniqueHash( object ):
         """ Return copy of `self`. """
         return UniqueHash( **{ k:v for k,v in self.__dict__.items() if not k[:1] == "_"} )
 
-    def __call__(self, *args, debug_trace : DebugTrace = None, **kwargs) -> str:
+    def __call__(__self__, # LEAVE THIS NAME. **kwargs might contain 'self' arguments.
+                 *args, debug_trace : DebugTrace = None, **kwargs) -> str:
         """
-        :meta public:
-            
-        Returns a unique hash for the `arg` and `kwargs` parameters passed to this function.
+        Returns a unique hash for the ``arg`` and ``kwargs`` parameters passed to this function.
         
         Example::
             
@@ -289,7 +288,7 @@ class UniqueHash( object ):
         args, kwargs:
             Parameters to hash.
             
-        debug_trace : :class:`cdxcore.uniquehash.DebugTrace`
+        debug_trace : :class:`cdxcore.uniquehash.DebugTrace` | None, default ``None``
             Allows tracing of hashing activity for debugging purposes.
             Two implementations of ``DebugTrace`` are available:
                 
@@ -302,13 +301,13 @@ class UniqueHash( object ):
         Returns
         -------
         Hash : str
-            String of at most `length`
+            String of at most ``self.length``.
         """
-        h, _ = self._mk_blake( h=self.length//2 )
+        h, _ = __self__._mk_blake( h=__self__.length//2 )
         if len(args) > 0:
-            self._hash_any( h, args, debug_trace = debug_trace )
+            __self__._hash_any( h, args, debug_trace = debug_trace )
         if len(kwargs) > 0:
-            self._hash_any( h, kwargs, debug_trace = debug_trace )
+            __self__._hash_any( h, kwargs, debug_trace = debug_trace )
         return h.hexdigest()
 
     # Utility functions

{cdxcore-0.1.27 → cdxcore-0.1.29}/cdxcore/version.py

@@ -655,7 +655,13 @@ def version( version              : str = "0.0.1" ,
                 if not gversion._class is None:
                     continue
                 gversion._class = f
-        f.version = Version(f, version, dep, auto_class=auto_class )
+                
+        version_ = Version(f, version, dep, auto_class=auto_class )
+        try:
+            f.version = version_
+        except AttributeError:
+            f.__dict__['version'] = version_
+        del version_
         assert type(f.version).__name__ == Version.__name__
         return f
     return wrap

{cdxcore-0.1.27 → cdxcore-0.1.29}/cdxcore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdxcore
-Version: 0.1.27
+Version: 0.1.29
 Summary: Basic Python Tools; upgraded cdxbasics
 Author-email: Hans Buehler <github@buehler.london>
 License-Expression: MIT

{cdxcore-0.1.27 → cdxcore-0.1.29}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdxcore"
-version = "0.1.27"
+version = "0.1.29"
 description = "Basic Python Tools; upgraded cdxbasics"
 authors = [{ name = "Hans Buehler", email = "github@buehler.london" }]
 readme = "README.md"

{cdxcore-0.1.27 → cdxcore-0.1.29}/tests/test_subdir.py

@@ -25,14 +25,15 @@ import_local()
 """
 Imports
 """
-from cdxcore.subdir import SubDir, CacheMode, VersionError, VersionPresentError
+from cdxcore.subdir import SubDir, CacheMode, VersionError, VersionPresentError, VersionedCacheRoot
+from cdxcore.version import version
 import numpy as np
 
 class Test(unittest.TestCase):
 
     def test_subdir(self):
 
-        sub = SubDir("!/.tmp_test_for_cdxbasics.subdir", delete_everything=True )
+        sub = SubDir("?/.tmp_test_for_cdxbasics.subdir", delete_everything=True )
         sub['y'] = 2
         sub.write('z',3)
         sub.write_string('l',"hallo")
@@ -193,11 +194,8 @@ class Test(unittest.TestCase):
         with self.assertRaises(VersionError):
             r = sub.read("test", None, version="2", raise_on_error=True)
             # wrong version
-        sub.delete_everything()
         
     def test_new(self):
-        
-        
         subdir = SubDir("my_directory")      # relative to current working directory
         subdir = SubDir("./my_directory")    # relative to current working directory
         subdir = SubDir("~/my_directory")    # relative to home directory
@@ -248,8 +246,6 @@ class Test(unittest.TestCase):
         test_format( SubDir.GZIP, True )
         test_format( SubDir.JSON_PICKLE )
 
-
-
     def test_cache_mode(self):
 
         on = CacheMode("on")
@@ -275,6 +271,146 @@ class Test(unittest.TestCase):
         self.assertEqual( [ x.write for x in allc ], [True, True, False, False, True, False] )
         self.assertEqual( [ x.delete for x in allc ], [False, False, False, True, True, False ] )
         self.assertEqual( [ x.del_incomp for x in allc ], [True, False, False, True, True, False ] )            
+
+class A(object):
+    def __init__(self, x):
+        self.x = x
+class B(object):
+    def __init__(self, x):
+        self.x = x
+     
+raw_sub = VersionedCacheRoot("?/subdir_cache_test2", exclude_arg_types=[A] )
+
+class C(object):
+    def __init__(self, x):
+        self.x = x
+    @raw_sub.cache("0.1")
+    def f(self, y):
+        return self.x*y
+        
+class Test(unittest.TestCase):
+
+    def test_cache( self ):
+        
+        sub = VersionedCacheRoot("?/subdir_cache_test", exclude_arg_types=[A] )
+        
+        @sub.cache("1.0")
+        def f(x):
+            return x
+        
+        _ = f(1)
+        _ = f(1)
+        self.assertTrue( f.cache_info.last_cached )
+
+        @sub.cache("1.0")
+        def g(x):
+            return x
+        
+        # a new B object each time -> no caching
+        _ = f(B(1))
+        _ = f(B(2))
+        self.assertFalse( f.cache_info.last_cached )
+
+        # same B object each time -> no caching
+        _ = f(B(1))
+        _ = f(B(1))
+        self.assertTrue( f.cache_info.last_cached )
+        
+        # a new A object each time -> caching, as we ignore 'A' types
+        _ = f(A(1))
+        _ = f(A(2))
+        self.assertTrue( f.cache_info.last_cached )
+        
+        @sub.cache("1.0", label=lambda x: f"f({x})")
+        def f(x):
+            return x
+        
+        _ = f(1)
+        self.assertEqual( f.cache_info.filename[:5], "f(1) " )
+        uid, _ = f(1, return_cache_uid=True)
+        self.assertEqual( uid[:5], "f(1) " )
+        
+        @sub.cache("1.0", uid=lambda x: f"f({x})")
+        def f(x):
+            return x
+        
+        _ = f(1)
+        self.assertEqual( f.cache_info.filename, "f(1)" )
+        uid, _ = f(1, return_cache_uid=True)
+        self.assertEqual( uid, "f(1)" )
+        
+        # test member caching
+        
+        c = C(2.)
+        _ = c.f(3)
+        self.assertFalse( c.f.cache_info.last_cached )
+        _ = c.f(3)
+        self.assertTrue( c.f.cache_info.last_cached )
+        _ = c.f(2)
+        self.assertFalse( c.f.cache_info.last_cached )
+        
+        # test versioning
+        
+        @version("F")
+        def F(x):
+            return x
+        
+        @sub.cache("G")
+        def G(x):
+            return x
+        
+        @sub.cache("H", dependencies=[F,G])
+        def H(x):
+            return G(x)*F(x)
+
+        _ = H(2.)
+        self.assertEqual( H.cache_info.version, "H { Test.test_cache.<locals>.F: F, Test.test_cache.<loc 3fabc694" )        
+        self.assertEqual( H.cache_info.version, H.version.unique_id64 )        
+
+        # decorate live member functions
+        
+        class AA(object):
+            def __init__(self,x):
+                self.x = x
+            def f(self,y):
+                return self.x*y
+        
+        a = AA(x=1)
+        f = sub.cache("0.1", label=lambda y : f"a.f({y})")(a.f)  # <- decorate bound 'f'.
+        _ = f(y=2)  
+        self.assertFalse( f.cache_info.last_cached )
+        self.assertEqual( f.cache_info.version, "0.1" )
+        _ = f(y=2)  
+        self.assertTrue( f.cache_info.last_cached )
+                
+        # funcname
+
+        sub = VersionedCacheRoot("?/subdir_cache_test", exclude_arg_types=[A], keep_last_arguments=True)
+        
+        @sub.cache("0.1", label=lambda new_func_name, func_name, x, y : f"{new_func_name}(): {func_name} {x} {y}", name_of_func_name_arg="new_func_name")
+        def f( func_name, x, y ):
+            pass
+        f("test",1,y=2)
+        self.assertEqual( repr(f.cache_info.arguments), "OrderedDict({'func_name': 'test', 'x': '1', 'y': '2'})" )
+        
+        # this should just work
+        @sub.cache("0.1", label=lambda func_name, x : f"{func_name} {x}")
+        def f( func_name, x ):
+            pass
+        with self.assertRaises(RuntimeError):
+            f("test",1)        
+            
+        # cuttinf off
+        @sub.cache("0.1", uid=lambda x,y: f"h2({x},{y})_______________________________________________________________________", exclude_args='debug') 
+        def h2(x,y,debug=False):
+            if debug:
+                print(f"h(x={x},y={y})")  
+            return x*y
+        h2(1,1)        
+# %%
+        self.assertEqual( h2.cache_info.filename, "h2(1,1)________________________________ 46a70d67" )
+
+        
 if __name__ == '__main__':
     unittest.main()
 

{cdxcore-0.1.27 → cdxcore-0.1.29}/tests/test_uniquehash.py

@@ -457,16 +457,25 @@ class Test(unittest.TestCase):
             def __unique_hash__( self, unique_hash, debug_trace ):
                 return ( self._seed, self._size )
             
+        class E(A):
+            """ Fixed string """
+            def __init__(self):
+                self.__unique_hash__ = "some_string"
+            
         empty = unique_hash( ('Test.test_uniqueHash.<locals>.A') )
         hash1 = unique_hash( A() )
         hash2 = unique_hash( B() )
         hash3 = unique_hash( C() )
         hash4 = unique_hash( D() )
+        hash5 = unique_hash( E() )
+        h_5   = unique_hash( E().__unique_hash__ )
         self.assertEqual( empty, "653a05ac14649dbceebbd7f3d4a0b89f" )        
         self.assertEqual( hash1, empty )
         self.assertEqual( hash2, "ae7cc6d56596eaa20dcd6aedc6e89d85" )
         self.assertEqual( hash3, "5e387f9e86426319577d2121a4e1437b" )
         self.assertEqual( hash4, "c9b449e95339458df155752acadbebb1" )
+        self.assertEqual( hash5, "79ee12b070c036e874263c6f4d70df98" )
+        self.assertEqual( hash5, h_5 )