experimaestro 0.22.0__py2.py3-none-any.whl → 0.23.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

experimaestro/__init__.py CHANGED
@@ -49,17 +49,19 @@ from .core.objects import (
49
49
  SerializedConfig,
50
50
  Serialized,
51
51
  )
52
+ from .core.types import Any, SubmitHook, submit_hook_decorator
53
+ from .launchers import Launcher
52
54
  from .scheduler.environment import Environment
53
55
  from .scheduler.workspace import Workspace, RunMode
54
56
  from .scheduler import Scheduler, experiment, FailedExperiment
55
57
  from .notifications import progress, tqdm
56
- from .core.types import Any
57
58
  from .checkers import Choices
58
59
  from .xpmutils import DirectoryContext
59
60
  from .mkdocs.annotations import documentation
61
+ from .scheduler.base import Job
60
62
 
61
63
 
62
- def set_launcher(launcher):
64
+ def set_launcher(launcher: Launcher):
63
65
  Workspace.CURRENT.launcher = launcher
64
66
 
65
67
 
experimaestro/__main__.py CHANGED
@@ -125,7 +125,7 @@ def diff(path: Path):
125
125
  """Show the reason of the identifier change for a job"""
126
126
  from experimaestro.tools.jobs import load_job
127
127
  from experimaestro import Config
128
- from experimaestro.core.objects import GenerationContext
128
+ from experimaestro.core.objects import ConfigWalkContext
129
129
 
130
130
  _, job = load_job(path / "params.json", discard_id=False)
131
131
  _, new_job = load_job(path / "params.json")
@@ -1,6 +1,5 @@
1
1
  """Command line jobs"""
2
2
 
3
- from collections import ChainMap
4
3
  import json
5
4
  import io
6
5
  from pathlib import Path
@@ -260,13 +259,6 @@ class CommandLineJob(Job):
260
259
 
261
260
  return None
262
261
 
263
- @property
264
- def environ(self):
265
- return ChainMap(
266
- self.workspace.environment.environ,
267
- self.launcher.environ if self.launcher else {},
268
- )
269
-
270
262
  @property
271
263
  def notificationURL(self):
272
264
  if self.launcher and self.launcher.notificationURL:
@@ -20,6 +20,7 @@ from typing import (
20
20
  List,
21
21
  Optional,
22
22
  Set,
23
+ Tuple,
23
24
  Type,
24
25
  TypeVar,
25
26
  Union,
@@ -35,6 +36,7 @@ from experimaestro.core.types import DeprecatedAttribute, ObjectType
35
36
  from .context import SerializationContext, SerializedPath, SerializedPathLoader
36
37
 
37
38
  if TYPE_CHECKING:
39
+ from experimaestro.scheduler.base import Job
38
40
  from experimaestro.scheduler.workspace import RunMode
39
41
  from experimaestro.launchers import Launcher
40
42
  from experimaestro.scheduler import Workspace
@@ -307,7 +309,7 @@ def add_to_path(p):
307
309
  sys.path = old_path
308
310
 
309
311
 
310
- class GenerationContext:
312
+ class ConfigWalkContext:
311
313
  """Context when generating values in configurations"""
312
314
 
313
315
  @property
@@ -342,33 +344,38 @@ class GenerationContext:
342
344
  NOT_SET = object()
343
345
 
344
346
 
345
- class ConfigProcessing:
347
+ class ConfigWalk:
346
348
  """Allows to perform an operation on all nested configurations"""
347
349
 
348
- def __init__(self, recurse_task=False):
350
+ def __init__(self, context: ConfigWalkContext = None, recurse_task=False):
349
351
  """
350
352
 
351
- Parameters:
352
- recurse_task: Recurse into linked tasks
353
+ :param recurse_task: Recurse into linked tasks
354
+ :param context: The context, by default only tracks the position in the
355
+ config tree
353
356
  """
354
357
  self.recurse_task = recurse_task
358
+ self.context = ConfigWalkContext() if context is None else context
355
359
 
356
360
  # Stores already visited nodes
357
361
  self.visited = {}
358
362
 
359
- def preprocess(self, config: "Config"):
363
+ def preprocess(self, config: "Config") -> Tuple[bool, Any]:
364
+ """Returns a tuple boolean/value
365
+
366
+ The boolean value is used to stop the processing if False.
367
+ The value is returned
368
+ """
360
369
  return True, None
361
370
 
362
371
  def postprocess(self, config: "Config", values: Dict[str, Any]):
363
372
  return config
364
373
 
365
- @contextmanager
366
374
  def list(self, i: int):
367
- yield i
375
+ return self.context.push(str(i))
368
376
 
369
- @contextmanager
370
377
  def map(self, k: str):
371
- yield k
378
+ return self.context.push(k)
372
379
 
373
380
  def __call__(self, x):
374
381
  if isinstance(x, Config):
@@ -426,18 +433,6 @@ class ConfigProcessing:
426
433
  raise NotImplementedError(f"Cannot handle a value of type {type(x)}")
427
434
 
428
435
 
429
- class GenerationConfigProcessing(ConfigProcessing):
430
- def __init__(self, context: GenerationContext, recurse_task=False):
431
- super().__init__(recurse_task=recurse_task)
432
- self.context = context
433
-
434
- def list(self, i: int):
435
- return self.context.push(str(i))
436
-
437
- def map(self, k: str):
438
- return self.context.push(k)
439
-
440
-
441
436
  def getqualattr(module, qualname):
442
437
  """Get a qualified attributed value"""
443
438
  cls = module
@@ -464,6 +459,7 @@ class ConfigInformation:
464
459
  # Meta-informations
465
460
  self._tags = {}
466
461
  self._initinfo = ""
462
+ self.submit_hooks = set()
467
463
 
468
464
  # Generated task
469
465
  self._taskoutput = None
@@ -534,7 +530,7 @@ class ConfigInformation:
534
530
  yield argument, self.values[argument.name]
535
531
 
536
532
  def tags(self):
537
- class TagFinder(ConfigProcessing):
533
+ class TagFinder(ConfigWalk):
538
534
  def __init__(self):
539
535
  super().__init__(recurse_task=True)
540
536
  self.tags = {}
@@ -578,7 +574,7 @@ class ConfigInformation:
578
574
  )
579
575
  raise
580
576
 
581
- def seal(self, context: GenerationContext):
577
+ def seal(self, context: ConfigWalkContext):
582
578
  """Seal the object, generating values when needed,
583
579
  before scheduling the associated job(s)
584
580
 
@@ -586,7 +582,7 @@ class ConfigInformation:
586
582
  - context: the generation context
587
583
  """
588
584
 
589
- class Sealer(GenerationConfigProcessing):
585
+ class Sealer(ConfigWalk):
590
586
  def preprocess(self, config: Config):
591
587
  return not config.__xpm__._sealed, config
592
588
 
@@ -607,9 +603,9 @@ class ConfigInformation:
607
603
 
608
604
  Internal API - do not use
609
605
  """
610
- context = GenerationContext()
606
+ context = ConfigWalkContext()
611
607
 
612
- class Unsealer(GenerationConfigProcessing):
608
+ class Unsealer(ConfigWalk):
613
609
  def preprocess(self, config: Config):
614
610
  return config.__xpm__._sealed, config
615
611
 
@@ -658,6 +654,23 @@ class ConfigInformation:
658
654
  logger.error("While setting %s", path + [argument.name])
659
655
  raise
660
656
 
657
+ def apply_submit_hooks(self, job: "Job"):
658
+ """Apply configuration hooks"""
659
+ context = ConfigWalkContext()
660
+
661
+ class HookGatherer(ConfigWalk):
662
+ def __init__(self, *args, **kwargs):
663
+ super().__init__(*args, **kwargs)
664
+ self.hooks = set()
665
+
666
+ def postprocess(self, config: "Config", values: Dict[str, Any]):
667
+ self.hooks.update(config.__xpm__.submit_hooks)
668
+
669
+ gatherer = HookGatherer(context, recurse_task=False)
670
+ gatherer(self.pyobject)
671
+ for hook in gatherer.hooks:
672
+ hook(job)
673
+
661
674
  def submit(
662
675
  self, workspace: "Workspace", launcher: "Launcher", run_mode=None
663
676
  ) -> "TaskOutput":
@@ -696,7 +709,7 @@ class ConfigInformation:
696
709
  experiment.CURRENT.workspace if experiment.CURRENT else None
697
710
  )
698
711
 
699
- # Call onSubmit
712
+ # Call onSubmit hooks
700
713
  launcher = (
701
714
  launcher
702
715
  or (workspace and workspace.launcher)
@@ -705,6 +718,9 @@ class ConfigInformation:
705
718
  if launcher:
706
719
  launcher.onSubmit(self.job)
707
720
 
721
+ # Apply submit hooks
722
+ self.apply_submit_hooks(self.job)
723
+
708
724
  # Add job dependencies
709
725
  self.updatedependencies(self.job.dependencies, [], set([id(self.pyobject)]))
710
726
 
@@ -1194,8 +1210,8 @@ class ConfigInformation:
1194
1210
 
1195
1211
  return o
1196
1212
 
1197
- class FromPython(GenerationConfigProcessing):
1198
- def __init__(self, context: GenerationContext):
1213
+ class FromPython(ConfigWalk):
1214
+ def __init__(self, context: ConfigWalkContext):
1199
1215
  super().__init__(context)
1200
1216
  self.objects = {}
1201
1217
 
@@ -1233,7 +1249,7 @@ class ConfigInformation:
1233
1249
 
1234
1250
  return o
1235
1251
 
1236
- def fromConfig(self, context: GenerationContext):
1252
+ def fromConfig(self, context: ConfigWalkContext):
1237
1253
  """Generate an instance given the current configuration"""
1238
1254
  self.validate()
1239
1255
  processor = ConfigInformation.FromPython(context)
@@ -1378,7 +1394,7 @@ class TypeConfig:
1378
1394
  self.__xpm__.add_dependencies(*dependencies)
1379
1395
  return self
1380
1396
 
1381
- def instance(self, context: GenerationContext = None) -> T:
1397
+ def instance(self, context: ConfigWalkContext = None) -> T:
1382
1398
  """Return an instance with the current values"""
1383
1399
  if context is None:
1384
1400
  from experimaestro.xpmutils import EmptyContext
@@ -1386,8 +1402,8 @@ class TypeConfig:
1386
1402
  context = EmptyContext()
1387
1403
  else:
1388
1404
  assert isinstance(
1389
- context, GenerationContext
1390
- ), f"{context.__class__} is not an instance of GenerationContext"
1405
+ context, ConfigWalkContext
1406
+ ), f"{context.__class__} is not an instance of ConfigWalkContext"
1391
1407
  return self.__xpm__.fromConfig(context) # type: ignore
1392
1408
 
1393
1409
  def submit(self, *, workspace=None, launcher=None, run_mode: "RunMode" = None):
@@ -1434,7 +1450,7 @@ class Config:
1434
1450
  configuration/task"""
1435
1451
 
1436
1452
  @classmethod
1437
- def __getxpmtype__(cls):
1453
+ def __getxpmtype__(cls) -> "ObjectType":
1438
1454
  """Get (and create if necessary) the Object type of this"""
1439
1455
  xpmtype = cls.__dict__.get("__xpmtype__", None)
1440
1456
  if xpmtype is None:
@@ -1,8 +1,10 @@
1
+ from abc import ABC
1
2
  import typing_extensions
2
3
 
3
4
  from experimaestro.core.types import ObjectType
4
5
  import experimaestro
5
6
  import io
7
+ from experimaestro.launchers import Launcher
6
8
  from experimaestro.scheduler.base import Job
7
9
 
8
10
  from experimaestro.scheduler.workspace import RunMode
@@ -20,7 +22,18 @@ from experimaestro.core.types import (
20
22
  from experimaestro.utils import logger as logger
21
23
  from functools import cached_property as cached_property
22
24
  from pathlib import Path
23
- from typing import Any, ClassVar, Dict, List, Optional, Set, TypeVar, Union, overload
25
+ from typing import (
26
+ Any,
27
+ Callable,
28
+ ClassVar,
29
+ Dict,
30
+ List,
31
+ Optional,
32
+ Set,
33
+ TypeVar,
34
+ Union,
35
+ overload,
36
+ )
24
37
 
25
38
  T = TypeVar("T", bound="Config")
26
39
 
@@ -62,7 +75,7 @@ class TaggedValue:
62
75
 
63
76
  def add_to_path(p) -> Generator[None, None, None]: ...
64
77
 
65
- class GenerationContext:
78
+ class ConfigWalkContext:
66
79
  @property
67
80
  def path(self) -> None: ...
68
81
  def __init__(self) -> None: ...
@@ -83,9 +96,9 @@ class ConfigProcessing:
83
96
  def map(self, k: str): ...
84
97
  def __call__(self, x): ...
85
98
 
86
- class GenerationConfigProcessing(ConfigProcessing):
99
+ class ConfigWalk(ConfigProcessing):
87
100
  context: Incomplete
88
- def __init__(self, context: GenerationContext) -> None: ...
101
+ def __init__(self, context: ConfigWalkContext) -> None: ...
89
102
  def list(self, i: int): ...
90
103
  def map(self, k: str): ...
91
104
 
@@ -108,7 +121,7 @@ class ConfigInformation:
108
121
  def xpmvalues(self, generated: bool = ...) -> Generator[Incomplete, None, None]: ...
109
122
  def tags(self): ...
110
123
  def validate(self) -> None: ...
111
- def seal(self, context: GenerationContext): ...
124
+ def seal(self, context: ConfigWalkContext): ...
112
125
  @property
113
126
  def identifier(self) -> Identifier: ...
114
127
  def dependency(self): ...
@@ -141,13 +154,13 @@ class ConfigInformation:
141
154
  save_directory: Optional[Path] = ...,
142
155
  ) -> Config: ...
143
156
 
144
- class FromPython(GenerationConfigProcessing):
157
+ class FromPython(ConfigWalk):
145
158
  objects: Incomplete
146
- def __init__(self, context: GenerationContext) -> None: ...
159
+ def __init__(self, context: ConfigWalkContext) -> None: ...
147
160
  def preprocess(self, config: Config): ...
148
161
  def postprocess(self, config: Config, values: Dict[str, Any]): ...
149
162
 
150
- def fromConfig(self, context: GenerationContext): ...
163
+ def fromConfig(self, context: ConfigWalkContext): ...
151
164
  def add_dependencies(self, *dependencies) -> None: ...
152
165
 
153
166
  def clone(v): ...
@@ -162,7 +175,7 @@ class TypeConfig:
162
175
  def __arguments__(self): ...
163
176
  def tags(self): ...
164
177
  def add_dependencies(self, *dependencies): ...
165
- def instance(self, context: GenerationContext = ...) -> T: ...
178
+ def instance(self, context: ConfigWalkContext = ...) -> T: ...
166
179
  def submit(
167
180
  self,
168
181
  *,
@@ -183,7 +196,7 @@ class Config:
183
196
  __xpmtype__: ClassVar[ObjectType]
184
197
  __xpm__: ConfigInformation
185
198
  @classmethod
186
- def __getxpmtype__(cls): ...
199
+ def __getxpmtype__(cls) -> ObjectType: ...
187
200
  def __getnewargs_ex__(self): ...
188
201
  @classmethod
189
202
  def c(cls, **kwargs) -> T: ...
@@ -1,6 +1,7 @@
1
+ from abc import ABC, abstractmethod
1
2
  import inspect
2
3
  import sys
3
- from typing import Union, Dict, Iterator, List
4
+ from typing import Set, Union, Dict, Iterator, List
4
5
  from collections import ChainMap
5
6
  from pathlib import Path
6
7
  import typing
@@ -17,6 +18,11 @@ if sys.version_info.major == 3 and sys.version_info.minor < 9:
17
18
  else:
18
19
  from typing import _AnnotatedAlias, get_type_hints
19
20
 
21
+ if typing.TYPE_CHECKING:
22
+ from experimaestro.scheduler.base import Job
23
+ from experimaestro.launchers import Launcher
24
+ from experimaestro.core.objects import Config
25
+
20
26
 
21
27
  class Identifier:
22
28
  def __init__(self, name: str):
@@ -143,7 +149,42 @@ class DeprecatedAttribute:
143
149
  self.fn(instance, value)
144
150
 
145
151
 
152
+ class SubmitHook(ABC):
153
+ """Hook called before the job is submitted to the scheduler
154
+
155
+ This allows modifying e.g. the run environnement
156
+ """
157
+
158
+ @abstractmethod
159
+ def __call__(self, job: "Job", launcher: "Launcher"):
160
+ ...
161
+
162
+ @abstractmethod
163
+ def __spec__(self):
164
+ """Returns an identifier tuple for hashing/equality"""
165
+ ...
166
+
167
+ def __eq__(self, other):
168
+ if other.__class__ is not self.__class__:
169
+ return False
170
+ return self.__spec__ == other.__spec__
171
+
172
+ def __hash__(self):
173
+ return hash((self.__class__, self.__spec__))
174
+
175
+
176
+ def submit_hook_decorator(hook: SubmitHook):
177
+ def decorator(cls: typing.Type["Config"]):
178
+ cls.__getxpmtype__().submit_hooks.add(hook)
179
+ return cls
180
+
181
+ return decorator
182
+
183
+
146
184
  class ObjectType(Type):
185
+ submit_hooks: Set[SubmitHook]
186
+ """Hooks associated with this configuration"""
187
+
147
188
  """ObjectType contains class-level information about
148
189
  experimaestro configurations and tasks
149
190
 
@@ -167,6 +208,7 @@ class ObjectType(Type):
167
208
  self.taskcommandfactory = None
168
209
  self.task = None
169
210
  self._title = None
211
+ self.submit_hooks = set()
170
212
 
171
213
  # Get the identifier
172
214
  if identifier is None and hasattr(tp, "__xpmid__"):
@@ -219,7 +261,7 @@ class ObjectType(Type):
219
261
  self.configtype.__module__ = tp.__module__
220
262
 
221
263
  # Create the type-specific object class
222
- # (now, the same as basetype - TODO: remove references)
264
+ # (now, the same as basetype - but in the future, remove references)
223
265
  self.objecttype = self.basetype # type: type
224
266
  self.basetype._ = self.configtype
225
267
 
@@ -1,15 +1,16 @@
1
1
  import inspect
2
2
  from pathlib import Path
3
- from typing import Callable, List, Tuple, Union
3
+ from typing import Callable, Union
4
4
  from experimaestro.core.arguments import ArgumentOptions, TypeAnnotation
5
- from experimaestro.core.objects import GenerationContext, Config
5
+ from experimaestro.core.objects import ConfigWalkContext, Config
6
6
 
7
7
 
8
8
  class Generator:
9
9
  """Base class for all generators"""
10
10
 
11
11
  def isoutput(self):
12
- """Returns True if this generator is a task output (e.g. generates a path within the job folder)"""
12
+ """Returns True if this generator is a task output (e.g. generates a
13
+ path within the job folder)"""
13
14
  return False
14
15
 
15
16
 
@@ -17,11 +18,11 @@ class PathGenerator(Generator):
17
18
  """Generates a path"""
18
19
 
19
20
  def __init__(
20
- self, path: Union[str, Path, Callable[[GenerationContext, Config], Path]]
21
+ self, path: Union[str, Path, Callable[[ConfigWalkContext, Config], Path]]
21
22
  ):
22
23
  self.path = path
23
24
 
24
- def __call__(self, context: GenerationContext, config: Config):
25
+ def __call__(self, context: ConfigWalkContext, config: Config):
25
26
  if inspect.isfunction(self.path):
26
27
  path = context.currentpath() / self.path(context, config) # type: Path
27
28
  else:
@@ -34,7 +35,7 @@ class PathGenerator(Generator):
34
35
 
35
36
 
36
37
  class pathgenerator(TypeAnnotation):
37
- def __init__(self, value: Union[str, Callable[[GenerationContext, Config], str]]):
38
+ def __init__(self, value: Union[str, Callable[[ConfigWalkContext, Config], str]]):
38
39
  self.value = value
39
40
 
40
41
  def annotate(self, options: ArgumentOptions):
@@ -1,25 +1,36 @@
1
1
  from pathlib import Path, PosixPath
2
- from typing import Callable, Dict, List, Optional, Protocol
2
+ from typing import Callable, Dict, List, Optional
3
3
  from experimaestro.commandline import AbstractCommand, Job, CommandLineJob
4
4
  from experimaestro.connectors import Connector
5
5
  from experimaestro.connectors.local import ProcessBuilder, LocalConnector
6
6
  from experimaestro.connectors.ssh import SshPath, SshConnector
7
+ from abc import ABC, abstractmethod
7
8
 
8
9
 
9
- class ScriptBuilder:
10
- lockfiles: List[Path]
11
- command: "AbstractCommand"
10
+ class ScriptBuilder(ABC):
12
11
  """A script builder is responsible for generating the script
13
12
  used to launch a command line job"""
14
13
 
14
+ lockfiles: List[Path]
15
+ """The files that must be locked before starting the job"""
16
+
17
+ command: "AbstractCommand"
18
+ """Command to be run"""
19
+
20
+ @abstractmethod
15
21
  def write(self, job: CommandLineJob) -> Path:
16
- raise NotImplementedError()
22
+ """Write the commmand line job
23
+
24
+ :params job: The job to be written
25
+ """
26
+ ...
17
27
 
18
28
 
19
29
  SubmitListener = Callable[[Job], None]
30
+ """Listen to job submissions"""
20
31
 
21
32
 
22
- class Launcher:
33
+ class Launcher(ABC):
23
34
  """A launcher"""
24
35
 
25
36
  submit_listeners: List[SubmitListener]
@@ -36,9 +47,10 @@ class Launcher:
36
47
  def setNotificationURL(self, url: Optional[str]):
37
48
  self.notificationURL = url
38
49
 
50
+ @abstractmethod
39
51
  def scriptbuilder(self) -> ScriptBuilder:
40
52
  """Returns a script builder"""
41
- raise NotImplementedError(f"For class {self.__class__}")
53
+ ...
42
54
 
43
55
  def addListener(self, listener: SubmitListener):
44
56
  self.submit_listeners.append(listener)
@@ -1,4 +1,4 @@
1
- from collections import defaultdict
1
+ from collections import ChainMap, defaultdict
2
2
  import os
3
3
  from pathlib import Path
4
4
  from shutil import rmtree
@@ -14,7 +14,7 @@ from experimaestro.scheduler.services import Service
14
14
  from experimaestro.settings import get_settings
15
15
 
16
16
 
17
- from experimaestro.core.objects import Config, GenerationContext
17
+ from experimaestro.core.objects import Config, ConfigWalkContext
18
18
  from experimaestro.utils import logger
19
19
  from experimaestro.locking import Locks, LockError, Lock
20
20
  from experimaestro.tokens import ProcessCounterToken
@@ -151,6 +151,23 @@ class Job(Resource):
151
151
  assert self._future, "Cannot wait a not submitted job"
152
152
  return self._future.result()
153
153
 
154
+ @property
155
+ def environ(self):
156
+ """Returns the job environment
157
+
158
+ It is made of (by order of priority):
159
+
160
+ 1. The job environment
161
+ 1. The launcher environment
162
+ 1. The workspace environment
163
+
164
+ """
165
+ return ChainMap(
166
+ {},
167
+ self.launcher.environ if self.launcher else {},
168
+ self.workspace.environment.environ,
169
+ )
170
+
154
171
  @property
155
172
  def progress(self):
156
173
  return self._progress
@@ -292,7 +309,7 @@ class Job(Resource):
292
309
  return self._future
293
310
 
294
311
 
295
- class JobContext(GenerationContext):
312
+ class JobContext(ConfigWalkContext):
296
313
  def __init__(self, job: Job):
297
314
  super().__init__()
298
315
  self.job = job
@@ -839,6 +856,7 @@ class experiment:
839
856
  def __enter__(self):
840
857
  logger.info("Locking experiment %s", self.xplockpath)
841
858
  self.xplock = self.workspace.connector.lock(self.xplockpath, 0).__enter__()
859
+ logger.info("Experiment locked")
842
860
 
843
861
  # Move old jobs into "jobs.bak"
844
862
  if self.workspace.run_mode == RunMode.NORMAL:
@@ -143,10 +143,15 @@ def proxy_response(base_url: str, request: Request, path: str):
143
143
 
144
144
 
145
145
  def start_app(server: "Server"):
146
+ logging.debug("Starting Flask server...")
146
147
  app = Flask("experimaestro")
147
- socketio = SocketIO(app, path="/api", async_mode="eventlet")
148
+
149
+ logging.debug("Starting Flask server (SocketIO)...")
150
+ socketio = SocketIO(app, path="/api", async_mode="gevent")
148
151
  listener = Listener(server.scheduler, socketio)
149
152
 
153
+ logging.debug("Starting Flask server (setting up socketio)...")
154
+
150
155
  @socketio.on("connect")
151
156
  def handle_connect():
152
157
  if server.token != request.cookies.get("experimaestro_token", None):
@@ -183,6 +188,8 @@ def start_app(server: "Server"):
183
188
  if process is not None:
184
189
  process.kill()
185
190
 
191
+ logging.debug("Starting Flask server (setting up routes)...")
192
+
186
193
  @app.route("/services/<path:path>", methods=["GET", "POST"])
187
194
  def route_service(path):
188
195
  service, *path = path.split("/", 1)
@@ -256,8 +263,9 @@ def start_app(server: "Server"):
256
263
 
257
264
  # Start the app
258
265
  if server.port is None or server.port == 0:
266
+ logging.info("Searching for an available port")
259
267
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
260
- sock.bind(("localhost", 0))
268
+ sock.bind(("", 0))
261
269
  server.port = sock.getsockname()[1]
262
270
  sock.close()
263
271
 
experimaestro/version.py CHANGED
@@ -1,4 +1,4 @@
1
1
  # file generated by setuptools_scm
2
2
  # don't change, don't track in version control
3
- __version__ = version = '0.22.0'
4
- __version_tuple__ = version_tuple = (0, 22, 0)
3
+ __version__ = version = '0.23.0'
4
+ __version_tuple__ = version_tuple = (0, 23, 0)
experimaestro/xpmutils.py CHANGED
@@ -1,10 +1,10 @@
1
1
  """Utilities exposed to users of the experimaestro API"""
2
2
 
3
3
  from pathlib import Path
4
- from experimaestro.core.objects import GenerationContext
4
+ from experimaestro.core.objects import ConfigWalkContext
5
5
 
6
6
 
7
- class DirectoryContext(GenerationContext):
7
+ class DirectoryContext(ConfigWalkContext):
8
8
  """Special generation context used for debugging and testing"""
9
9
 
10
10
  def __init__(self, path: Path):
@@ -16,7 +16,7 @@ class DirectoryContext(GenerationContext):
16
16
  return self._path
17
17
 
18
18
 
19
- class EmptyContext(GenerationContext):
19
+ class EmptyContext(ConfigWalkContext):
20
20
  """Special generation context used for debugging and testing"""
21
21
 
22
22
  @property
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: experimaestro
3
- Version: 0.22.0
3
+ Version: 0.23.0
4
4
  Summary: "Experimaestro is a computer science experiment manager"
5
5
  Home-page: https://github.com/experimaestro/experimaestro-python
6
6
  Author: Benjamin Piwowarski
@@ -32,7 +32,7 @@ Requires-Dist: sortedcontainers
32
32
  Requires-Dist: pyparsing
33
33
  Requires-Dist: humanfriendly
34
34
  Requires-Dist: huggingface-hub (~=0.11.1)
35
- Requires-Dist: eventlet
35
+ Requires-Dist: gevent
36
36
  Requires-Dist: flask
37
37
  Requires-Dist: flask-socketio
38
38
  Requires-Dist: Arpeggio (>=2.0)
@@ -40,7 +40,7 @@ Requires-Dist: watchdog (>2.0.0)
40
40
  Requires-Dist: marshmallow
41
41
  Requires-Dist: fabric
42
42
  Requires-Dist: rpyc
43
- Requires-Dist: typing-extensions (>=3.7.4.3) ; python_version < "3.9"
43
+ Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.11"
44
44
  Requires-Dist: cached-property ; python_version < "3.9"
45
45
  Provides-Extra: dev
46
46
  Requires-Dist: docutils ; extra == 'dev'
@@ -155,6 +155,12 @@ if __name__ == "__main__":
155
155
 
156
156
  which can be launched with `python test.py /tmp/helloworld-workdir`
157
157
 
158
+ ## 0.23.0 (2023-04-07)
159
+
160
+ ### Feat
161
+
162
+ - submit hooks to allow e.g. changing the environment variables
163
+
158
164
  ## 0.22.0 (2023-04-05)
159
165
 
160
166
  ### Feat
@@ -1,12 +1,12 @@
1
- experimaestro/__init__.py,sha256=kaK8hWjIxRexeEQ27L5cReExgG2tgvZjvaOFrHMJEg4,1294
2
- experimaestro/__main__.py,sha256=A_hT19SXNCOIRaW27sfFMwT0fdvgglQo2jPFJQQQwYM,11192
1
+ experimaestro/__init__.py,sha256=FvCfQt53icBffG2vqLb9zTdBkB04cQhLbiabPmY1yVs,1403
2
+ experimaestro/__main__.py,sha256=DcrhIXrMACuPp40CO-PAKTytHhPaozQwqnGIQ8NfoHs,11192
3
3
  experimaestro/annotations.py,sha256=laaAy2C57SJTPeRuD0B5jRUN0TgZYckO3fjBIR9laCs,8603
4
4
  experimaestro/checkers.py,sha256=ZCMbnE_GFC5compWjt-fuHhPImi9fCPjImF8Ow9NqK8,696
5
5
  experimaestro/click.py,sha256=HTfjm4poqfG69MIYztrPWEei5OVnd154UCr2pJm6yp8,2390
6
- experimaestro/commandline.py,sha256=YUpAb_z65fSMs0TToNig2xLObfI-OFu2c95iLAbAXmA,9677
6
+ experimaestro/commandline.py,sha256=NS1ubme8DTJtDS2uWwdHLQiZsl6TSK1LkNxu39c3-cw,9463
7
7
  experimaestro/compat.py,sha256=dQqE2ZNHLM2wtdfp7fBRYMfC33qNehVf9J6FGRBUQhs,171
8
8
  experimaestro/filter.py,sha256=Qp8ObI3fBGYJjNo2TMu7N3d1rVTdYXP9PVBLFg5JUHk,5775
9
- experimaestro/generators.py,sha256=AUXaGA5Hvl-agkxybkJ8dQtFGW0yIJzuC3X8WU_Gf2g,1235
9
+ experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
10
10
  experimaestro/huggingface.py,sha256=2f0MqNxfth90R7kO6VVJ3K5tXdx95w2d1ljggrL6l_E,3056
11
11
  experimaestro/ipc.py,sha256=0sVzK8mZ4VCRQ5L3piCqTm1o-AVfTfJEXW26P6SOFrY,1694
12
12
  experimaestro/locking.py,sha256=hPT-LuDGZTijpbme8O0kEoB9a3WjdVzI2h31OT44UxE,1477
@@ -19,23 +19,23 @@ experimaestro/settings.py,sha256=E6nFyrUB6uBYQlQARczr_YKUn06zRoZ_RbPgDj6jHaU,914
19
19
  experimaestro/taskglobals.py,sha256=aBjPpo4HQp6E6M3GQ8L6PR4rK2Lu0kD5dS1WjnaGgDc,499
20
20
  experimaestro/tokens.py,sha256=aHT6lN4YLF0ujXl0nNu1sSSx-2ebrYiZFfrFvvmsQ1s,14681
21
21
  experimaestro/typingutils.py,sha256=gkEhJ9Fir05xh5v45pYKmrGYiLCykGtlU8sL-ydFXS8,1785
22
- experimaestro/version.py,sha256=sm9RIoEcmkvSrNOjZY--ysY4sf77dk0DEKPYtk1caog,162
23
- experimaestro/xpmutils.py,sha256=E3Q9ZMMi2L30J-KxHE0nzt-pbW9Nehu6Fxy_-7qdln0,638
22
+ experimaestro/version.py,sha256=Ktf2waZZK1xthVXoT6iRjDhQfoED_kinKyheuED-ztk,162
23
+ experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
24
24
  experimaestro/connectors/__init__.py,sha256=hxcBSeVLk_7oyiIlS3l-9dGg1NGtShwvRD1tS7f8D2M,5461
25
25
  experimaestro/connectors/local.py,sha256=w4IN_pxiPh9jb5u4yIH569ZbVHb6i2LfGhdsGFQ625o,5760
26
26
  experimaestro/connectors/ssh.py,sha256=rQv_j3yQ5g-HyX4tvO_E0qD6xkrZyTq5oRYQEC1WhYU,8206
27
27
  experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
28
  experimaestro/core/arguments.py,sha256=KzvPfO92deWsgjhrPULjFuNpr8aCDMbLhCoz5xafrtg,5824
29
29
  experimaestro/core/context.py,sha256=70ZhDBobGDl0wEyvZyqyrnWagLs7EiRf0cBSXp1N3Lc,2479
30
- experimaestro/core/objects.py,sha256=p6Ojws0GZByxlYYCcvPUacMiJWYYzyvlxFUuQPNf6t4,55390
31
- experimaestro/core/objects.pyi,sha256=EUXDaIuj5_V3zZPU6ODsw5hZ4d4B4MbNYNnAv2x9-3E,7753
32
- experimaestro/core/types.py,sha256=M3XvJNnRFgquLgbogB8zCnInWeapig_yYzrCbizRaKw,17972
30
+ experimaestro/core/objects.py,sha256=IijmRTPTVMsUKHgVugOJam-Ml_ms83vFfy4C1igN6mc,56141
31
+ experimaestro/core/objects.pyi,sha256=vMy3EWCSwZNky8zr5i80C9lAkx4ON8Y46dcBNQpdE58,7855
32
+ experimaestro/core/types.py,sha256=f8y-N0hKsqxgX_l__Vh5xIh4tudw2Ws2RkECHB3Ki44,19099
33
33
  experimaestro/launcherfinder/__init__.py,sha256=sM9U0dDvUkyRIEv5QnU9XZCn5uJ6Q6x4OAbWPmN9qcs,148
34
34
  experimaestro/launcherfinder/base.py,sha256=NptPJ0e8CktdhOPejocSfI_B4mloeH_EmJrbXruUCSA,1020
35
35
  experimaestro/launcherfinder/parser.py,sha256=0qDXgdPce_PsWDy-hKTfxxjXjTAu4FA8moKtyllB2-Q,2129
36
36
  experimaestro/launcherfinder/registry.py,sha256=Ng50ONPcu-BKtxN5z6rjSg7FPHMVva3wekdKvRSam4U,7363
37
37
  experimaestro/launcherfinder/specs.py,sha256=xedeGt-Qgz9X2A5zaVWborsUsFp2XSTmRK6pJIMrJ9A,5359
38
- experimaestro/launchers/__init__.py,sha256=ahVRwtBS7uFlkNC1JQ6xHnx0C18ZeJ21J7Imx9i2Hec,2305
38
+ experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4OzRnjHXg,2525
39
39
  experimaestro/launchers/direct.py,sha256=jxvlBLp-s1NiZ91kNUzGfnkdkHAwsrDZYPBDu_rX6dU,1883
40
40
  experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
41
  experimaestro/launchers/slurm.py,sha256=50c0K_yuLUmFAkl1KH755yDKYvc0aIha1jOQY38IcwI,24121
@@ -45,12 +45,12 @@ experimaestro/mkdocs/base.py,sha256=SwLh9s7BZfrTAZdBaealSqVeLAroDSwLLMOHmLCxMPQ,
45
45
  experimaestro/mkdocs/metaloader.py,sha256=qCqnTWhlgxql-oe46E8AbvYdoM311-lQh-msmPnbllQ,1481
46
46
  experimaestro/mkdocs/style.css,sha256=42kJ6Ozq_n4Iw5UfJ4-nO1u-HN3ELvV7Vhvj1Xkn7rQ,66
47
47
  experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
48
- experimaestro/scheduler/base.py,sha256=vq8lxD3E5CmhZGYNncTrmiu1RWx8x0lMR3LjKz-agY4,29136
48
+ experimaestro/scheduler/base.py,sha256=ZFHIEabBfunuSqQN4lffs8VcJCG07MImyFaKLYc3MKo,29588
49
49
  experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
50
50
  experimaestro/scheduler/environment.py,sha256=ZaSHSgAcZBmIj7b_eS1OvNQOuVLFvuw-qvqtYrc3Vms,2393
51
51
  experimaestro/scheduler/services.py,sha256=lrxhM4feQuGjtBVHJbEdmzI9x-bPBa5rXVcaDjjiSqU,1820
52
52
  experimaestro/scheduler/workspace.py,sha256=xATJi6-GJcpdwB4alliJnmAuvwt-URUiUKUfq5scUac,1731
53
- experimaestro/server/__init__.py,sha256=AF0zy5caEUqSkRxQrRcaAoioGx_qhY-kGgqOuRVGca0,10135
53
+ experimaestro/server/__init__.py,sha256=0Dy3cdpaBOun7VwNvXwHmGdR13Hc4DQBDQyQsF45430,10420
54
54
  experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
55
55
  experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
56
56
  experimaestro/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -106,9 +106,9 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
106
106
  experimaestro/utils/resources.py,sha256=gDjkrRjo7GULWyXmNXm_u1uqzEIAoAvJydICk56nOQw,1006
107
107
  experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
108
108
  experimaestro/utils/yaml.py,sha256=jEjqXqUtJ333wNUdIc0o3LGvdsTQ9AKW9a9CCd-bmGU,6766
109
- experimaestro-0.22.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
110
- experimaestro-0.22.0.dist-info/METADATA,sha256=cv_KTbAUynHNfBWfmBW3553Qd1ynovAcryGIyGH7-B0,6381
111
- experimaestro-0.22.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
112
- experimaestro-0.22.0.dist-info/entry_points.txt,sha256=VsHkdvdBt9feNcnofvFyNbEDNjKU3BTSVPJcAGWNgko,591
113
- experimaestro-0.22.0.dist-info/top_level.txt,sha256=siYS2iOls_-f2iUoulKaaXuQQHroIDN36S8YAI1CFZk,14
114
- experimaestro-0.22.0.dist-info/RECORD,,
109
+ experimaestro-0.23.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
110
+ experimaestro-0.23.0.dist-info/METADATA,sha256=SK-V-y7c9HJeXvhxyicyC2k9u8C6POr_0Jlu4ycZ1Ds,6475
111
+ experimaestro-0.23.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
112
+ experimaestro-0.23.0.dist-info/entry_points.txt,sha256=VsHkdvdBt9feNcnofvFyNbEDNjKU3BTSVPJcAGWNgko,591
113
+ experimaestro-0.23.0.dist-info/top_level.txt,sha256=siYS2iOls_-f2iUoulKaaXuQQHroIDN36S8YAI1CFZk,14
114
+ experimaestro-0.23.0.dist-info/RECORD,,