experimaestro 1.11.1__tar.gz → 2.0.0rc0__tar.gz

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.11.1
+Version: 2.0.0rc0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "typing-extensions >=4.2; python_version < \"3.12\"",
     "watchdog >=2"
 ]
-version = "1.11.1"
+version = "2.0.0-rc0"
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -83,6 +83,12 @@ poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-timeout>=2.4.0",
+]
+
 [tool.poetry.group.ssh]
 optional = true
 
@@ -140,7 +146,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.11.1"
+version = "2.0.0a0"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/src/experimaestro/annotations.py

@@ -71,7 +71,7 @@ class config:
         assert inspect.isclass(tp), f"{tp} is not a class"
 
         # Adds to xpminfo for on demand creation of information
-        return ObjectType(tp, identifier=self.identifier).basetype
+        return ObjectType(tp, identifier=self.identifier).valuetype
 
 
 class Array(TypeProxy):

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/src/experimaestro/cli/__init__.py

@@ -20,16 +20,6 @@ from experimaestro.settings import find_workspace
 logging.basicConfig(level=logging.INFO)
 
 
-def pass_cfg(f):
-    """Pass configuration information"""
-
-    @click.pass_context
-    def new_func(ctx, *args, **kwargs):
-        return ctx.invoke(f, ctx.obj, *args, **kwargs)
-
-    return update_wrapper(new_func, f)
-
-
 def check_xp_path(ctx, self, path: Path):
     if not (path / ".__experimaestro__").is_file():
         cprint(f"{path} is not an experimaestro working directory", "red")
@@ -142,7 +132,6 @@ def diff(path: Path):
     """Show the reason of the identifier change for a job"""
     from experimaestro.tools.jobs import load_job
     from experimaestro import Config
-    from experimaestro.core.objects import ConfigWalkContext
 
     _, job = load_job(path / "params.json", discard_id=False)
     _, new_job = load_job(path / "params.json")
@@ -290,6 +279,16 @@ cli.add_command(Launchers("launchers", help="Launcher specific commands"))
 cli.add_command(Launchers("connectors", help="Connector specific commands"))
 cli.add_command(Launchers("tokens", help="Token specific commands"))
 
+# Import and add progress commands
+from .progress import progress as progress_cli
+
+cli.add_command(progress_cli)
+
+# Import and add jobs commands
+from .jobs import jobs as jobs_cli
+
+cli.add_command(jobs_cli)
+
 
 @cli.group()
 @click.option("--workdir", type=Path, default=None)

experimaestro-2.0.0rc0/src/experimaestro/cli/progress.py

@@ -0,0 +1,269 @@
+"""Simplified CLI commands for managing and viewing progress files"""
+
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict
+
+import click
+from termcolor import colored
+
+try:
+    from tqdm import tqdm
+
+    TQDM_AVAILABLE = True
+except ImportError:
+    TQDM_AVAILABLE = False
+
+from experimaestro.progress import ProgressEntry, ProgressFileReader
+from experimaestro.settings import find_workspace
+from . import cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def progress(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Progress tracking commands"""
+    ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+
+
+def format_timestamp(timestamp: float) -> str:
+    """Format timestamp for display"""
+    dt = datetime.fromtimestamp(timestamp)
+    return dt.strftime("%Y-%m-%d %H:%M:%S")
+
+
+@click.argument("jobid", type=str)
+@progress.command()
+@click.pass_context
+def show(ctx, jobid: str):
+    """Show current progress state (default command)
+
+    JOBID format: task_name/task_hash
+    """
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    current_progress = reader.get_current_progress()
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    # Filter out EOJ markers
+    current_progress = {k: v for k, v in current_progress.items() if k != -1}
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    click.echo(f"Progress for job {jobid}")
+    click.echo("=" * 80)
+
+    # Show simple text-based progress for each level
+    for level in sorted(current_progress.keys()):
+        entry = current_progress[level]
+        indent = "  " * level
+        progress_pct = f"{entry.progress * 100:5.1f}%"
+        desc = entry.desc or f"Level {level}"
+        timestamp = format_timestamp(entry.timestamp)
+
+        color = "green" if entry.progress >= 1.0 else "yellow"
+        click.echo(colored(f"{indent}L{level}: {progress_pct} - {desc}", color))
+        click.echo(colored(f"{indent}     Last updated: {timestamp}", "cyan"))
+
+
+def create_progress_bar(
+    level: int,
+    desc: str,
+    progress: float = 0.0,
+) -> tqdm:
+    """Create a properly aligned progress bar like dashboard style"""
+    if level > 0:
+        indent = "   " * (level - 1) + "└─ "
+    else:
+        indent = ""
+    label = f"{indent}L{level}"
+
+    colors = ["blue", "yellow", "magenta", "cyan", "white"]
+    bar_color = colors[level % len(colors)]
+
+    unit = desc[:50] if desc else f"Level {level}"
+    ncols = 100
+    wbar = 50
+
+    return tqdm(
+        total=100,
+        desc=label,
+        position=level,
+        leave=True,
+        bar_format=f"{{desc}}: {{percentage:3.0f}}%|{{bar:{wbar - len(indent)}}}| {{unit}}",  # noqa: F541
+        ncols=ncols,  # Adjust width based on level
+        unit=unit,
+        colour=bar_color,
+        initial=progress * 100,
+    )
+
+
+def _update_progress_display(
+    reader: ProgressFileReader, progress_bars: Dict[int, tqdm]
+) -> bool:
+    """Update the tqdm progress bars in dashboard style"""
+    current_state: Dict[int, ProgressEntry] = {
+        k: v for k, v in reader.get_current_state().items() if k != -1
+    }
+
+    if not current_state:
+        click.echo("No progress information available yet...")
+        return False
+
+    # Update existing bars and create new ones
+    for _level, entry in current_state.items():
+        progress_val = entry.progress * 100
+        desc = entry.desc or f"Level {entry.level}"
+
+        if entry.level not in progress_bars:
+            progress_bars[entry.level] = create_progress_bar(
+                entry.level, desc, progress_val
+            )
+
+        bar = progress_bars[entry.level]
+        bar.unit = desc[:50]
+        bar.n = progress_val
+
+        bar.refresh()
+
+    # Remove bars for levels that no longer exist
+    levels_to_remove = set(progress_bars.keys()) - set(current_state.keys())
+    for level in levels_to_remove:
+        progress_bars[level].close()
+        del progress_bars[level]
+
+    return True
+
+
+@click.argument("jobid", type=str)
+@click.option("--refresh-rate", "-r", default=0.5, help="Refresh rate in seconds")
+@progress.command()
+@click.pass_context
+def live(ctx, jobid: str, refresh_rate: float):
+    """Show live progress with tqdm-style bars
+
+    JOBID format: task_name/task_hash
+    """
+    if not TQDM_AVAILABLE:
+        click.echo("tqdm is not available. Install with: pip install tqdm")
+        click.echo("Falling back to basic display...")
+        ctx.invoke(show, jobid=jobid)
+        return
+
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    progress_bars: Dict[int, tqdm] = {}
+
+    def cleanup_bars():
+        """Clean up all progress bars"""
+        for bar in progress_bars.values():
+            bar.close()
+        progress_bars.clear()
+
+    click.echo(f"Live progress for job {jobid}")
+    click.echo("Press Ctrl+C to stop")
+    click.echo("=" * 80)
+
+    try:
+        if not _update_progress_display(reader, progress_bars):
+            click.echo("No progress information available yet...")
+
+        while True:
+            time.sleep(refresh_rate)
+
+            if not _update_progress_display(reader, progress_bars):
+                # Check if job is complete
+                if reader.is_done():
+                    click.echo("\nJob completed!")
+                    break
+
+            # Check if all progress bars are at 100%
+            if progress_bars and all(bar.n >= 100 for bar in progress_bars.values()):
+                cleanup_bars()
+                click.echo("\nAll progress completed!")
+                break
+
+    except KeyboardInterrupt:
+        click.echo("\nStopped monitoring progress")
+    finally:
+        cleanup_bars()
+
+
+@progress.command(name="list")
+@click.pass_context
+def list_jobs(ctx):
+    """List all jobs with progress information"""
+    ws = ctx.obj.workspace
+    jobs_path = ws.path / "jobs"
+
+    if not jobs_path.exists():
+        click.echo("No jobs directory found")
+        return
+
+    for task_dir in jobs_path.iterdir():
+        if not task_dir.is_dir():
+            continue
+
+        for job_dir in task_dir.iterdir():
+            if not job_dir.is_dir():
+                continue
+
+            progress_dir = job_dir / ".experimaestro"
+            if not progress_dir.exists():
+                continue
+
+            # Check if there are progress files
+            progress_files = list(progress_dir.glob("progress-*.jsonl"))
+            if not progress_files:
+                continue
+
+            job_id = f"{task_dir.name}/{job_dir.name}"
+            reader = ProgressFileReader(job_dir)
+            current_state = reader.get_current_state()
+
+            # if current_progress:
+            if current_state:
+                # Get overall progress (level 0)
+                level_0 = current_state.get(0)
+                if level_0:
+                    color = "green" if level_0.progress >= 1.0 else "yellow"
+                    desc = f"{level_0.desc}" if level_0.desc else ""
+                    progress_pct = f"{level_0.progress * 100:5.1f}%"
+                    click.echo(colored(f"{job_id:50} - {progress_pct} - {desc}", color))
+
+                else:
+                    click.echo(f"{job_id:50} No level 0 progress")
+            else:
+                click.echo(f"{job_id:50} No progress data")

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/src/experimaestro/core/identifier.py

@@ -170,7 +170,7 @@ class IdentifierComputer:
             self._hashupdate(IdentifierComputer.ENUM_ID)
             k = value.__class__
             self._hashupdate(
-                f"{k.__module__}.{k.__qualname__ }:{value.name}".encode("utf-8"),
+                f"{k.__module__}.{k.__qualname__}:{value.name}".encode("utf-8"),
             )
         elif isinstance(value, dict):
             self._hashupdate(IdentifierComputer.DICT_ID)
@@ -235,7 +235,16 @@ class IdentifierComputer:
                 # Skip if the argument is not a constant, and
                 # - optional argument: both value and default are None
                 # - the argument value is equal to the default value
-                argvalue = getattr(value, argument.name, None)
+                try:
+                    argvalue = getattr(value, argument.name, None)
+                except KeyError:
+                    logging.warning(
+                        "Parameter %s has not been set in %s created at %s",
+                        argument.name,
+                        value,
+                        value.__xpm__._initinfo,
+                    )
+                    raise
                 if not argument.constant and (
                     (
                         not argument.required

{experimaestro-1.11.1 → experimaestro-2.0.0rc0}/src/experimaestro/core/objects/config.py

@@ -25,7 +25,6 @@ from typing import (
     Optional,
     Set,
     Tuple,
-    Type,
     TypeVar,
     Union,
     overload,
@@ -122,11 +121,11 @@ class ConfigInformation:
     def __init__(self, pyobject: "ConfigMixin"):
         # The underlying pyobject and XPM type
         self.pyobject = pyobject
-        self.xpmtype = pyobject.__xpmtype__  # type: ObjectType
+        self.xpmtype: "ObjectType" = pyobject.__xpmtype__
         self.values = {}
 
         # Meta-informations
-        self._tags = {}
+        self._tags: dict[str, Any] = {}
         self._initinfo = ""
 
         self._taskoutput = None
@@ -142,7 +141,7 @@ class ConfigInformation:
         #: True when this configuration was loaded from disk
         self.loaded = False
 
-        # Explicitely added dependencies
+        # Explicitly added dependencies
         self.dependencies = []
 
         # Concrete type variables resolutions
@@ -170,6 +169,11 @@ class ConfigInformation:
         self._sealed = False
         self._meta = None
 
+        #: This flags is True when a value in this configuration,
+        #: or any sub-configuration, is generated. This prevents problem
+        #: when a configuration with generated values is re-used
+        self._has_generated_value = False
+
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
         assert not self._sealed, "Configuration is sealed"
@@ -198,6 +202,20 @@ class ConfigInformation:
         if self._sealed and not bypass:
             raise AttributeError(f"Object is read-only (trying to set {k})")
 
+        if not isinstance(v, ConfigMixin) and isinstance(v, Config):
+            raise AttributeError(
+                "Configuration (and not objects) should be used. Consider using .C(...)"
+            )
+
+        if (
+            isinstance(v, ConfigMixin)
+            and v.__xpm__._has_generated_value
+            and v.__xpm__.task is None
+        ):
+            raise AttributeError(
+                f"Cannot set {k} to a configuration with generated values"
+            )
+
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
@@ -326,12 +344,20 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
+        subconfigs = [
+            v.__xpm__
+            for v in self.values.values()
+            if isinstance(v, Config) and v.__xpm__.task is None
+        ]
+
+        if any(v._has_generated_value for v in subconfigs):
+            raise AttributeError("Cannot seal a configuration with generated values")
 
         class Sealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return not config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 # Generate values
                 from experimaestro.generators import Generator
 
@@ -344,6 +370,7 @@ class ConfigInformation:
                                     continue
                                 value = argument.generator()
                             else:
+                                # Generate a value
                                 sig = inspect.signature(argument.generator)
                                 if len(sig.parameters) == 0:
                                     value = argument.generator()
@@ -354,12 +381,23 @@ class ConfigInformation:
                                         False
                                     ), "generator has either two parameters (context and config), or none"
                             config.__xpm__.set(k, value, bypass=True)
+                            config.__xpm__._has_generated_value = True
+                        else:
+                            value = config.__xpm__.values.get(k)
                     except Exception:
                         logger.error(
                             "While setting %s of %s", argument.name, config.__xpmtype__
                         )
                         raise
 
+                    # Propagate the generated value flag
+                    if (
+                        (value is not None)
+                        and isinstance(value, ConfigMixin)
+                        and value.__xpm__._has_generated_value
+                    ):
+                        self._has_generated_value = True
+
                 config.__xpm__._sealed = True
 
         Sealer(context, recurse_task=True)(self.pyobject)
@@ -657,13 +695,6 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration # FIXME: remove
-        def mark_output(config: "Config"):
-            """Sets a dependency on the job"""
-            assert not isinstance(config, Task), "Cannot set a dependency on a task"
-            config.__xpm__.task = self.pyobject
-            return config
-
         # Mark this configuration also
         self.task = self.pyobject
 
@@ -677,6 +708,9 @@ class ConfigInformation:
     def mark_output(self, config: "Config"):
         """Sets a dependency on the job"""
         assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        assert isinstance(
+            config, ConfigMixin
+        ), "Only configurations can be marked as dependent on a task"
         config.__xpm__.task = self.pyobject
         return config
 
@@ -762,7 +796,7 @@ class ConfigInformation:
         state_dict = {
             "id": id(self.pyobject),
             "module": self.xpmtype._module,
-            "type": self.xpmtype.basetype.__qualname__,
+            "type": self.xpmtype.value_type.__qualname__,
             "typename": self.xpmtype.name(),
             "identifier": self.identifier.state_dict(),
         }
@@ -1022,7 +1056,7 @@ class ConfigInformation:
 
             # Creates an object (or a config)
             if as_instance:
-                o = cls.XPMValue.__new__(cls.XPMValue)
+                o = cls.__new__(cls)
             else:
                 o = cls.XPMConfig.__new__(cls.XPMConfig)
             assert definition["id"] not in objects, "Duplicate id %s" % definition["id"]
@@ -1183,7 +1217,7 @@ class ConfigInformation:
 
             if o is None:
                 # Creates an object (and not a config)
-                o = config.XPMValue()
+                o = config.__xpmtype__.value_type()
 
                 # Store in cache
                 self.objects.add_stub(id(config), o)
@@ -1242,6 +1276,9 @@ def clone(v):
     if isinstance(v, Enum):
         return v
 
+    if isinstance(v, tuple):
+        return tuple(clone(x) for x in v)
+
     if isinstance(v, Config):
         # Create a new instance
         kwargs = {
@@ -1260,6 +1297,11 @@ class ConfigMixin:
     """Class for configuration objects"""
 
     __xpmtype__: ObjectType
+    """The associated XPM type"""
+
+    __xpm__: ConfigInformation
+    """The __xpm__ object contains all instance specific information about a
+    configuration/task"""
 
     def __init__(self, **kwargs):
         """Initialize the configuration with the given parameters"""
@@ -1310,8 +1352,8 @@ class ConfigMixin:
             [f"{key}={value}" for key, value in self.__xpm__.values.items()]
         )
         return (
-            f"{self.__xpmtype__.basetype.__module__}."
-            f"{self.__xpmtype__.basetype.__qualname__}({params})"
+            f"{self.__xpmtype__.value_type.__module__}."
+            f"{self.__xpmtype__.value_type.__qualname__}({params})"
         )
 
     def tag(self, name, value):
@@ -1397,6 +1439,9 @@ class ConfigMixin:
         return clone(self)
 
     def add_pretasks(self, *tasks: "LightweightTask"):
+        assert all(
+            [isinstance(task, ConfigMixin) for task in tasks]
+        ), "One of the parameters is not a configuration object"
         assert all(
             [isinstance(task, LightweightTask) for task in tasks]
         ), "One of the pre-tasks are not lightweight tasks"
@@ -1441,52 +1486,17 @@ class Config:
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
 
-    __xpm__: ConfigInformation
-    """The __xpm__ object contains all instance specific information about a
-    configuration/task"""
-
     @classproperty
     def XPMConfig(cls):
         if issubclass(cls, ConfigMixin):
             return cls
-        return cls.__getxpmtype__().configtype
-
-    @classproperty
-    def XPMValue(cls):
-        """Returns the value object for this configuration"""
-        if issubclass(cls, ConfigMixin):
-            return cls.__xpmtype__.objecttype
-
-        if value_cls := cls.__dict__.get("__XPMValue__", None):
-            pass
-        else:
-            from ..types import XPMValue
-
-            __objectbases__ = tuple(
-                s.XPMValue
-                for s in cls.__bases__
-                if issubclass(s, Config) and (s is not Config)
-            ) or (XPMValue,)
-
-            *tp_qual, tp_name = cls.__qualname__.split(".")
-            value_cls = type(f"{tp_name}.XPMValue", (cls,) + __objectbases__, {})
-            value_cls.__qualname__ = ".".join(tp_qual + [value_cls.__name__])
-            value_cls.__module__ = cls.__module__
-
-            setattr(cls, "__XPMValue__", value_cls)
-
-        return value_cls
+        return cls.__getxpmtype__().config_type
 
     @classproperty
     def C(cls):
         """Alias for XPMConfig"""
         return cls.XPMConfig
 
-    @classproperty
-    def V(cls):
-        """Alias for XPMValue"""
-        return cls.XPMValue
-
     @classmethod
     def __getxpmtype__(cls) -> "ObjectType":
         """Get (and create if necessary) the Object type associated
@@ -1503,46 +1513,6 @@ class Config:
                 raise
         return xpmtype
 
-    def __new__(cls: Type[T], *args, **kwargs) -> T:
-        """Returns an instance of a ConfigMixin (for compatibility, use XPMConfig
-        or C if possible)
-
-        :deprecated: Use Config.C or Config.XPMConfig to construct a new
-            configuration, and Config.V (or Config.XPMValue) for a new value
-        """
-        # If this is an XPMValue, just return a new instance
-        from experimaestro.core.types import XPMValue
-
-        if issubclass(cls, XPMValue):
-            return object.__new__(cls)
-
-        # If this is the XPMConfig, just return a new instance
-        # __init__ will be called
-        if issubclass(cls, ConfigMixin):
-            return object.__new__(cls)
-
-        # Log a deprecation warning for this way of creating a configuration
-        caller = inspect.getframeinfo(inspect.stack()[1][0])
-        logger.warning(
-            "Creating a configuration using Config.__new__ is deprecated, and will be removed in a future version. "
-            "Use Config.C or Config.XPMConfig to create a new configuration. "
-            "Issue created at %s:%s",
-            str(Path(caller.filename).absolute()),
-            caller.lineno,
-        )
-
-        # otherwise, we use the configuration type
-        o: ConfigMixin = object.__new__(cls.__getxpmtype__().configtype)
-        try:
-            o.__init__(*args, **kwargs)
-        except Exception:
-            logger.error(
-                "Init error in %s:%s"
-                % (str(Path(caller.filename).absolute()), caller.lineno)
-            )
-            raise
-        return o
-
     def __validate__(self):
         """Validate the values"""
         pass