experimaestro 2.0.0a4__tar.gz → 2.0.0b13__tar.gz

experimaestro-2.0.0b13/.gitignore

@@ -0,0 +1,18 @@
+dist
+*.pyc
+*.egg-info
+.vscode/
+.tox
+**/.DS_Store
+
+__pycache__/
+.mypy_cache/
+.pytest_cache
+test-reports/
+.eggs
+site/
+build/
+
+# Generated
+src/experimaestro/version.py
+src/experimaestro/webui/data

{experimaestro-2.0.0a4 → experimaestro-2.0.0b13}/PKG-INFO

@@ -1,61 +1,92 @@
 Metadata-Version: 2.4
 Name: experimaestro
-Version: 2.0.0a4
-Summary: "Experimaestro is a computer science experiment manager"
-License: GPL-3
+Version: 2.0.0b13
+Summary: Experimaestro is a computer science experiment manager
+Project-URL: Homepage, https://github.com/experimaestro/experimaestro-python
+Project-URL: Documentation, https://experimaestro-python.readthedocs.io/
+Project-URL: Repository, https://github.com/experimaestro/experimaestro-python
+Project-URL: Bug Tracker, https://github.com/experimaestro/experimaestro-python/issues
+Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
+License: GPL-3.0-or-later
 License-File: LICENSE
 Keywords: experiment manager
-Author: Benjamin Piwowarski
-Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: arpeggio (>=2,<3)
-Requires-Dist: attrs (>=23.1.0,<24)
-Requires-Dist: click (>=8)
-Requires-Dist: decorator (>=5,<6)
-Requires-Dist: docstring-parser (>=0.15,<1)
-Requires-Dist: fasteners (>=0.19,<1)
-Requires-Dist: flask (>=2.3,<3)
-Requires-Dist: flask-socketio (>=5.3,<6)
-Requires-Dist: gevent (>=25)
-Requires-Dist: gevent-websocket (>=0.10)
-Requires-Dist: huggingface-hub (>0.17)
-Requires-Dist: humanfriendly (>=10)
-Requires-Dist: marshmallow (>=3.20,<4)
-Requires-Dist: mkdocs (>=1.5,<2)
-Requires-Dist: omegaconf (>=2.3,<3)
-Requires-Dist: psutil (>=7,<8)
-Requires-Dist: pyparsing (>=3.1,<4)
-Requires-Dist: pytools (>=2023.1.1,<2024)
-Requires-Dist: pyyaml (>=6.0.1,<7)
-Requires-Dist: requests (>=2.31,<3)
-Requires-Dist: rpyc (>=5,<7)
-Requires-Dist: sortedcontainers (>=2.4,<3)
-Requires-Dist: termcolor (>=2.3,<3)
-Requires-Dist: tqdm (>=4.66.1,<5)
-Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.12"
-Requires-Dist: watchdog (>=2)
+Requires-Python: >=3.10
+Requires-Dist: arpeggio<3,>=2
+Requires-Dist: attrs<24,>=23.1.0
+Requires-Dist: click>=8
+Requires-Dist: decorator<6,>=5
+Requires-Dist: docstring-parser<1,>=0.15
+Requires-Dist: fastapi<1,>=0.109
+Requires-Dist: fasteners<1,>=0.19
+Requires-Dist: httpx<1,>=0.26
+Requires-Dist: huggingface-hub>0.17
+Requires-Dist: humanfriendly>=10
+Requires-Dist: marshmallow<4,>=3.20
+Requires-Dist: omegaconf<3,>=2.3
+Requires-Dist: peewee<4,>=3.17
+Requires-Dist: psutil<8,>=7
+Requires-Dist: pyparsing<4,>=3.1
+Requires-Dist: pyperclip<2,>=1.8
+Requires-Dist: pytools<2024,>=2023.1.1
+Requires-Dist: pyyaml<7,>=6.0.1
+Requires-Dist: requests<3,>=2.31
+Requires-Dist: rpyc<7,>=5
+Requires-Dist: sortedcontainers<3,>=2.4
+Requires-Dist: termcolor<3,>=2.3
+Requires-Dist: textual>=6
+Requires-Dist: tqdm<5,>=4.66.1
+Requires-Dist: typing-extensions>=4.2; python_version < '3.12'
+Requires-Dist: uvicorn[standard]<1,>=0.27
+Requires-Dist: watchdog>=2
+Provides-Extra: dev
+Requires-Dist: docutils>=0.18; extra == 'dev'
+Requires-Dist: mypy>=1.0; extra == 'dev'
+Requires-Dist: pygments>=2.15; extra == 'dev'
+Requires-Dist: pytest-timeout>=2.4.0; extra == 'dev'
+Requires-Dist: pytest>=8.4.1; extra == 'dev'
+Requires-Dist: textual-dev>=1.8.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: myst-parser>=2.0; extra == 'docs'
+Requires-Dist: sphinx-codeautolink>=0.15; extra == 'docs'
+Requires-Dist: sphinx-copybutton>=0.5; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme>=2.0; extra == 'docs'
+Requires-Dist: sphinx>=6; extra == 'docs'
+Provides-Extra: ssh
+Requires-Dist: fabric>=3; extra == 'ssh'
+Requires-Dist: paramiko>=3.3; extra == 'ssh'
 Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/experimaestro.svg)](https://badge.fury.io/py/experimaestro)
 [![RTD](https://readthedocs.org/projects/experimaestro-python/badge/?version=latest)](https://experimaestro-python.readthedocs.io)
 
-Experimaestro helps in designing and managing complex workflows. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:
+Experimaestro helps in designing and managing **complex experimental plans**. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:
 
 - **Task Automation**: The tool automates repetitive tasks, making it easier to run large-scale experiments. It's particularly useful in scenarios where experiments need to be repeated with different parameters or datasets.
 - **Resource Management**: It efficiently manages computational resources, which is critical when dealing with data-intensive tasks or when running multiple experiments in parallel.
-- **Extensibility**: Experimaestro is designed to be flexible and extensible, allowing users to integrate it with various programming languages and tools commonly used in data science and research.
-- **Reproducibility**: By keeping a detailed record of experiments, including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.
-- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows.
+- **Reproducibility**: By keeping a detailed record of experiments (the experimental plan in python), including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.
+- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows (web and text-based).
+
+The full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io). A tutorial (training a CNN on MNIST) is [available on github](https://github.com/experimaestro/experimaestro-demo).
+
+# Screenshots
 
-The full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io)
+## Textual interface (new in v2)
 
+![Experiment screen](docs/source/img/tui-experiments.png)
+
+![Jobs screen](docs/source/img/tui-jobs.png)
+
+![Log screen](docs/source/img/tui-logs.png)
 
 # Install
 
@@ -150,4 +181,3 @@ if __name__ == "__main__":
 ```
 
 which can be launched with `python test.py /tmp/helloworld-workdir`
-

{experimaestro-2.0.0a4 → experimaestro-2.0.0b13}/README.md

@@ -1,16 +1,24 @@
 [![PyPI version](https://badge.fury.io/py/experimaestro.svg)](https://badge.fury.io/py/experimaestro)
 [![RTD](https://readthedocs.org/projects/experimaestro-python/badge/?version=latest)](https://experimaestro-python.readthedocs.io)
 
-Experimaestro helps in designing and managing complex workflows. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:
+Experimaestro helps in designing and managing **complex experimental plans**. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:
 
 - **Task Automation**: The tool automates repetitive tasks, making it easier to run large-scale experiments. It's particularly useful in scenarios where experiments need to be repeated with different parameters or datasets.
 - **Resource Management**: It efficiently manages computational resources, which is critical when dealing with data-intensive tasks or when running multiple experiments in parallel.
-- **Extensibility**: Experimaestro is designed to be flexible and extensible, allowing users to integrate it with various programming languages and tools commonly used in data science and research.
-- **Reproducibility**: By keeping a detailed record of experiments, including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.
-- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows.
+- **Reproducibility**: By keeping a detailed record of experiments (the experimental plan in python), including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.
+- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows (web and text-based).
 
-The full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io)
+The full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io). A tutorial (training a CNN on MNIST) is [available on github](https://github.com/experimaestro/experimaestro-demo).
 
+# Screenshots
+
+## Textual interface (new in v2)
+
+![Experiment screen](docs/source/img/tui-experiments.png)
+
+![Jobs screen](docs/source/img/tui-jobs.png)
+
+![Log screen](docs/source/img/tui-logs.png)
 
 # Install
 

experimaestro-2.0.0b13/pyproject.toml

@@ -0,0 +1,167 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "experimaestro"
+authors = [
+    {name = "Benjamin Piwowarski", email = "benjamin@piwowarski.fr"}
+]
+description = "Experimaestro is a computer science experiment manager"
+readme = "README.md"
+license = {text = "GPL-3.0-or-later"}
+keywords = ["experiment manager"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.10"
+dynamic = ["version"]
+dependencies = [
+    "arpeggio >=2,<3",
+    "attrs >=23.1.0,<24",
+    "click >=8",
+    "decorator >=5,<6",
+    "docstring-parser >=0.15,<1",
+    "fastapi >=0.109,<1",
+    "fasteners >=0.19,<1",
+    "httpx >=0.26,<1",
+    "humanfriendly >=10",
+    "huggingface-hub >0.17",
+    "marshmallow >=3.20,<4",
+    "omegaconf >=2.3,<3",
+    "peewee >=3.17,<4",
+    "psutil >=7,<8",
+    "pyparsing >=3.1,<4",
+    "pyperclip >=1.8,<2",
+    "pytools >=2023.1.1,<2024",
+    "pyyaml >=6.0.1,<7",
+    "requests >=2.31,<3",
+    "rpyc >=5,<7",
+    "sortedcontainers >=2.4,<3",
+    "termcolor >=2.3,<3",
+    "textual >=6",
+    "tqdm >=4.66.1,<5",
+    "typing-extensions >=4.2; python_version < \"3.12\"",
+    "uvicorn[standard] >=0.27,<1",
+    "watchdog >=2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "mypy>=1.0",
+    "pytest>=8.4.1",
+    "pytest-timeout>=2.4.0",
+    "textual-dev>=1.8.0",
+    "docutils>=0.18",
+    "Pygments>=2.15",
+]
+docs = [
+    "sphinx>=6",
+    "myst-parser>=2.0",
+    "sphinx-rtd-theme>=2.0",
+    "sphinx-codeautolink>=0.15",
+    "sphinx-copybutton>=0.5",
+]
+ssh = [
+    "paramiko>=3.3",
+    "fabric>=3",
+]
+
+[project.urls]
+Homepage = "https://github.com/experimaestro/experimaestro-python"
+Documentation = "https://experimaestro-python.readthedocs.io/"
+Repository = "https://github.com/experimaestro/experimaestro-python"
+"Bug Tracker" = "https://github.com/experimaestro/experimaestro-python/issues"
+
+[project.scripts]
+experimaestro = "experimaestro.__main__:main"
+
+[project.entry-points."experimaestro.process"]
+local = "experimaestro.connectors.local:LocalProcess"
+slurm = "experimaestro.launchers.slurm:BatchSlurmProcess"
+
+[project.entry-points."experimaestro.connectors"]
+local = "experimaestro.connectors.local:LocalConnector"
+ssh = "experimaestro.connectors.ssh:SshConnector"
+
+[project.entry-points."experimaestro.tokens"]
+unix = "experimaestro.tokens:CounterToken"
+
+[project.entry-points.mypy]
+experimaestro = "experimaestro.mypy:plugin"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/experimaestro/version.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+    "/README.md",
+    "/LICENSE",
+    "/pyproject.toml",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/experimaestro"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"src/experimaestro/webui/data" = "experimaestro/webui/data"
+"src/experimaestro/sphinx/static/experimaestro.css" = "experimaestro/sphinx/static/experimaestro.css"
+"src/experimaestro/tui/app.tcss" = "experimaestro/tui/app.tcss"
+
+[tool.pydocstyle]
+match-dir = '(?!tests)(?!resources)(?!docs)[^\.].*'
+match = '(?!test)(?!setup)[^\._].*\.py'
+inherit = false
+ignore = "D200,D203,D213,D406,D407"
+
+[tool.mypy]
+python_version = "3.10"
+warn_unused_ignores = true
+
+[tool.pytest.ini_options]
+junit_family = "xunit2"
+testpaths = ["src/experimaestro"]
+norecursedirs = ["node_modules"]
+log_format = "[%(levelname)s] %(asctime)s %(name)s [%(process)d/%(threadName)s]: %(message)s"
+log_date_format = "%H:%M:%S"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.8",
+    "mkdocs>=1.6.1",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "F",      # pyflakes
+    "W",      # pycodestyle warnings
+    "C90",    # mccabe complexity
+    "T20",    # flake8-print
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 25

{experimaestro-2.0.0a4 → experimaestro-2.0.0b13}/src/experimaestro/__init__.py

@@ -4,15 +4,7 @@ from pathlib import Path
 
 # Annotations
 from .annotations import (
-    config,
-    task,
-    param,
-    ConstantParam,
-    constant,
-    option,
-    pathoption,
     cache,
-    Identifier,
     Array,
     TagDict,
     tag,
@@ -21,12 +13,11 @@ from .annotations import (
     STDOUT,
     STDERR,
     deprecate,
-    # deprecated
-    argument,
     initializer,
     # Method
     config_only,
 )
+from .core.types import Identifier
 from .core.serialization import (
     load,
     save,
@@ -47,15 +38,22 @@ from .core.arguments import (
     field,
     # Annotations helpers
     help,
-    default,
 )
 from .generators import pathgenerator, PathGenerator
+from .core.partial import (
+    partial,
+    param_group,
+    ParameterGroup,
+    Partial,
+)
 from .core.objects import (
     Config,
+    InstanceConfig,
     copyconfig,
     setmeta,
     DependentMarker,
     Task,
+    ResumableTask,
     LightweightTask,
     ObjectStore,
 )
@@ -63,8 +61,10 @@ from .core.context import SerializationContext
 from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
-from .scheduler import Scheduler, experiment, FailedExperiment
+from .scheduler import Scheduler, experiment, FailedExperiment, DirtyGitError
+from .exceptions import GracefulTimeout
 from .scheduler.workspace import Workspace, RunMode
+from .scheduler.transient import TransientMode
 from .scheduler.state import get_experiment
 from .notifications import progress, tqdm
 from .checkers import Choices
@@ -72,6 +72,7 @@ from .xpmutils import DirectoryContext
 from .mkdocs.annotations import documentation
 from .scheduler.base import Job
 from .launcherfinder.registry import LauncherRegistry
+from .experiments.configuration import DirtyGitAction
 
 
 def set_launcher(launcher: Launcher):