experimaestro 2.0.0a1__py3-none-any.whl → 2.0.0a3__py3-none-any.whl

experimaestro/core/arguments.py

@@ -80,10 +80,13 @@ class Argument:
 
         self.generator = generator
         self.default = None
+        self.ignore_generated = False
 
         if default is not None:
             assert self.generator is None, "generator and default are exclusive options"
             if isinstance(default, field):
+                self.ignore_generated = default.ignore_generated
+
                 if default.default is not None:
                     self.default = default.default
                 elif default.default_factory is not None:
@@ -184,13 +187,29 @@ DataPath = Annotated[Path, dataHint]
 class field:
     """Extra information for a given experimaestro field (param or meta)"""
 
-    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+    def __init__(
+        self,
+        *,
+        default: Any = None,
+        default_factory: Callable = None,
+        ignore_generated=False,
+    ):
+        """Gives some extra per-field information
+
+        :param default: a default value, defaults to None
+        :param default_factory: a default factory for values, defaults to None
+        :param ignore_generated: True if the value is hidden – it won't be accessible in
+            tasks, defaults to False. The interest of hidden is to add a
+            configuration field that changes the identifier, but will not be
+            used.
+        """
         assert not (
             (default is not None) and (default_factory is not None)
         ), "default and default_factory are mutually exclusive options"
 
         self.default_factory = default_factory
         self.default = default
+        self.ignore_generated = ignore_generated
 
 
 class help(TypeAnnotation):

experimaestro/core/objects/config.py

@@ -169,10 +169,33 @@ class ConfigInformation:
         self._sealed = False
         self._meta = None
 
-        #: This flags is True when a value in this configuration,
-        #: or any sub-configuration, is generated. This prevents problem
-        #: when a configuration with generated values is re-used
-        self._has_generated_value = False
+        # This contains the list of generated values (using context) in this
+        # configuration or any sub-configuration, is generated. This prevents
+        # problem when a configuration with generated values is re-used.
+        self._generated_values = []
+
+    def get_generated_paths(
+        self, path: list[str] = None, paths: list[str] = None
+    ) -> list[str]:
+        """Get the list of generated paths, useful to track down those
+
+        :param path: The current path
+        :param paths: The list of generated paths so far, defaults to None
+        :return: The full list of generated paths
+        """
+        paths = [] if paths is None else paths
+        path = [] if path is None else path
+
+        for key in self._generated_values:
+            value = self.values[key]
+            if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
+                path.append(key)
+                value.__xpm__.get_generated_paths(path, paths)
+                path.pop()
+            else:
+                paths.append(".".join(path + [key]))
+
+        return paths
 
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
@@ -207,18 +230,21 @@ class ConfigInformation:
                 "Configuration (and not objects) should be used. Consider using .C(...)"
             )
 
-        if (
-            isinstance(v, ConfigMixin)
-            and v.__xpm__._has_generated_value
-            and v.__xpm__.task is None
-        ):
-            raise AttributeError(
-                f"Cannot set {k} to a configuration with generated values"
-            )
-
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
+                if (
+                    isinstance(v, ConfigMixin)
+                    and v.__xpm__._generated_values
+                    and v.__xpm__.task is None
+                    and not argument.ignore_generated
+                ):
+                    raise AttributeError(
+                        f"Cannot set {k} to a configuration with generated values. "
+                        "Here is the list of paths to help you: "
+                        f"""{', '.join(v.__xpm__.get_generated_paths([k]))}"""
+                    )
+
                 if not bypass and (
                     (isinstance(argument.generator, Generator)) or argument.constant
                 ):
@@ -344,14 +370,17 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
-        subconfigs = [
-            v.__xpm__
-            for v in self.values.values()
-            if isinstance(v, Config) and v.__xpm__.task is None
-        ]
-
-        if any(v._has_generated_value for v in subconfigs):
-            raise AttributeError("Cannot seal a configuration with generated values")
+        if generated_keys := [
+            k
+            for k, v in self.values.items()
+            if isinstance(v, Config)
+            and v.__xpm__.task is None
+            and v.__xpm__._generated_values
+        ]:
+            raise AttributeError(
+                "Cannot seal a configuration with generated values:"
+                f"""{",".join(generated_keys)} in {context.currentpath}"""
+            )
 
         class Sealer(ConfigWalk):
             def preprocess(self, config: ConfigMixin):
@@ -375,13 +404,18 @@ class ConfigInformation:
                                 if len(sig.parameters) == 0:
                                     value = argument.generator()
                                 elif len(sig.parameters) == 2:
+                                    # Only in that case do we need to flag this configuration
+                                    # as containing generated values
+                                    if not argument.ignore_generated:
+                                        config.__xpm__._generated_values.append(k)
+                                    else:
+                                        logging.warning("Ignoring %s", k)
                                     value = argument.generator(self.context, config)
                                 else:
                                     assert (
                                         False
                                     ), "generator has either two parameters (context and config), or none"
                             config.__xpm__.set(k, value, bypass=True)
-                            config.__xpm__._has_generated_value = True
                         else:
                             value = config.__xpm__.values.get(k)
                     except Exception:
@@ -392,11 +426,14 @@ class ConfigInformation:
 
                     # Propagate the generated value flag
                     if (
-                        (value is not None)
+                        value is not None
                         and isinstance(value, ConfigMixin)
-                        and value.__xpm__._has_generated_value
+                        and value.__xpm__._generated_values
                     ):
-                        self._has_generated_value = True
+                        if not argument.ignore_generated:
+                            config.__xpm__._generated_values.append(k)
+                        else:
+                            logging.warning("Ignoring %s", k)
 
                 config.__xpm__._sealed = True
 

experimaestro/core/objects/config_walk.py

@@ -71,6 +71,7 @@ class ConfigWalk:
         return self.context.push(str(i))
 
     def map(self, k: str):
+        """Provides a path context when processing a tree"""
         return self.context.push(k)
 
     def stub(self, config):
@@ -123,7 +124,8 @@ class ConfigWalk:
                 and self.recurse_task
                 and x.__xpm__.task is not x
             ):
-                self(x.__xpm__.task)
+                with self.map("__task__"):
+                    self(x.__xpm__.task)
 
             processed = self.postprocess(stub, x, result)
             self.visited[xid] = processed

experimaestro/tests/test_generators.py

@@ -14,6 +14,10 @@ class Learner(Task):
     x: Param[int]
 
 
+class ModuleLoader(Task):
+    validation: Param[Validation] = field(ignore_generated=True)
+
+
 def test_generators_reuse_on_submit():
     # We have one way to select the best model
     validation = Validation.C()
@@ -59,3 +63,6 @@ def test_generators_reuse_on_set():
         # We should not be able to *create* a second task with the same validation,
         # even without submitting it
         Learner.C(x=2, validation=validation)
+
+    # This should run OK
+    ModuleLoader.C(validation=validation)

{experimaestro-2.0.0a1.dist-info → experimaestro-2.0.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 2.0.0a1
+Version: 2.0.0a3
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-2.0.0a1.dist-info → experimaestro-2.0.0a3.dist-info}/RECORD

@@ -13,14 +13,14 @@ experimaestro/connectors/__init__.py,sha256=UKhDU3uv9jFH37oUb0JiejrekA85xtEirn79
 experimaestro/connectors/local.py,sha256=lCGIubqmUJZ1glLtLRXOgakTMfEaEmFtNkEcw9qV5vw,6143
 experimaestro/connectors/ssh.py,sha256=5giqvv1y0QQKF-GI0IFUzI_Z5H8Bj9EuL_Szpvk899Q,8600
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-experimaestro/core/arguments.py,sha256=7hpkU1f8LJ7JL8kQaD514h9CFSfMotYLsVfMsMmdpWk,6487
+experimaestro/core/arguments.py,sha256=gB0Kq9XL0_mYbm9WHL-mDx2tUGHI1pvsu1ahMaDxIY0,7170
 experimaestro/core/callbacks.py,sha256=59JfeUgWcCCdIQ3pvh-xNnoRp9BX8f4iOAkgm16wBzE,1660
 experimaestro/core/context.py,sha256=1tLmX7WcgEKSbGw77vfziTzS8KNsoZJ02JBWMBCqqOk,2606
 experimaestro/core/identifier.py,sha256=d-DczyKvQhqyGD9I1ndHrPoOFRDcBHNzeqQx6EOrDPo,10552
 experimaestro/core/objects/__init__.py,sha256=ucJY5e17QQ1Kc-GYXeL7g8GFj8rP0XB4g2vrl32uhxY,721
-experimaestro/core/objects/config.py,sha256=VUdkAup9jcHhl53pfeNDvuqc6KTnvr4yrGktnaR8aRw,56574
+experimaestro/core/objects/config.py,sha256=jgaGYBiT3V3yEUgVxcpOdX3mozylL78eYSC0woy0664,58308
 experimaestro/core/objects/config_utils.py,sha256=ZLECGkeIWdzunm8vwWsQhvcSgV1e064BgXbLiZnxSEM,1288
-experimaestro/core/objects/config_walk.py,sha256=gyDMrVPbBMChn7r4em_gQXuqnxASO_JVauEbnJNO8II,4245
+experimaestro/core/objects/config_walk.py,sha256=b8u6oohf1gXyva4Y_Cyyl_3BNivzI2y-I2B6MUPV2aU,4353
 experimaestro/core/objects.pyi,sha256=xvlsRj4u1xsJxbevJl5Ner_HwmxR8x1JlAeIVDJzuy0,6498
 experimaestro/core/serialization.py,sha256=CSPEwOzlDsgAz6V2og-TgyU0RXDtzt_nXaoXFZleDZE,5775
 experimaestro/core/serializers.py,sha256=R_CAMyjjfU1oi-eHU6VlEUixJpFayGqEPaYu7VsD9xA,1197
@@ -125,7 +125,7 @@ experimaestro/tests/test_file_progress.py,sha256=uNUUq-ptcnxu56zRvUUZ5EYM9ZIQbUm
 experimaestro/tests/test_file_progress_integration.py,sha256=ejXASpdnpi6HUy569Q5Vx5F8SV_RDU-4o3yuQka6vS4,16751
 experimaestro/tests/test_findlauncher.py,sha256=KPy8ow--NXS1KFCIpxrmEJFRvjo-v-PwlVHVyoVKLPg,3134
 experimaestro/tests/test_forward.py,sha256=9y1zYm7hT_Lx5citxnK7n20cMZ2WJbsaEeY5irCZ9U4,735
-experimaestro/tests/test_generators.py,sha256=GDPNaB7-ZNTOe5hyC5E54jzq8a4XNRNszJkxCO0BR_s,2011
+experimaestro/tests/test_generators.py,sha256=rj4oQBxcW_UXeyykQ37QetKora-bxDJEY5AGkBaWEWI,2172
 experimaestro/tests/test_identifier.py,sha256=PfUK-IYXCTyK1YMrsphkQx4BmsST5Z3bWSNgu-ZameU,14284
 experimaestro/tests/test_instance.py,sha256=QaJAxZzyUQiEwx5lmfJKDw4vg1p6ASPhR6QJPXWjhnw,1705
 experimaestro/tests/test_objects.py,sha256=hGku35h1qkNMIdgP_gWM7HeviaqW7jrZDffOsCJh-_Q,1787
@@ -156,8 +156,8 @@ experimaestro/utils/multiprocessing.py,sha256=am3DkHP_kmWbpynbck2c9QystCUtPBoSAC
 experimaestro/utils/resources.py,sha256=j-nvsTFwmgENMoVGOD2Ap-UD3WU85WkI0IgeSszMCX4,1328
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-2.0.0a1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-2.0.0a1.dist-info/METADATA,sha256=65IJsCiXen3sKUqSzV4vBL_1l8JziDkI2ytAp3lDLmU,5690
-experimaestro-2.0.0a1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-experimaestro-2.0.0a1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-2.0.0a1.dist-info/RECORD,,
+experimaestro-2.0.0a3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-2.0.0a3.dist-info/METADATA,sha256=rdyvmeG6mIEMPcvZARoSrXDuBt2NX6bEPUufnfyRzhE,5690
+experimaestro-2.0.0a3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+experimaestro-2.0.0a3.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-2.0.0a3.dist-info/RECORD,,