experimaestro 1.7.0rc0__tar.gz → 1.7.0rc2__tar.gz

{experimaestro-1.7.0rc0 → experimaestro-1.7.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.7.0rc0
+Version: 1.7.0rc2
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
@@ -34,6 +34,7 @@ Requires-Dist: huggingface-hub (>0.17)
 Requires-Dist: humanfriendly (>=10,<11)
 Requires-Dist: marshmallow (>=3.20,<4.0)
 Requires-Dist: omegaconf (>=2.3,<3.0)
+Requires-Dist: protobuf (>5)
 Requires-Dist: psutil (>=7)
 Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)

{experimaestro-1.7.0rc0 → experimaestro-1.7.0rc2}/pyproject.toml

@@ -20,7 +20,7 @@ include = [
     "src/experimaestro/mkdocs/style.css",
     { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.7.0-rc0"
+version = "1.7.0-rc2"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -29,7 +29,10 @@ enable = false
 vcs = "git"
 style = "semver"
 dirty = true
-format-jinja = """{{ serialize_semver(base, ["rc" + revision|string]) }}"""
+format-jinja = """{%- set pre = [] -%}{%- set metadata = [] -%}
+{%- if revision is not none -%}{{ pre.append("rc" + revision|string) or "" }}{%- endif -%}
+{%- if distance > 0 -%}{{ metadata.append(distance|string) or "" }}{%- endif -%}
+{{ serialize_semver(base, pre, metadata)}}"""
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
@@ -61,6 +64,7 @@ watchdog = "^2"
 marshmallow = "^3.20"
 decorator = "^5"
 rpyc = ">=5,<7"
+protobuf = ">5" 
 
 [tool.poetry.group.ssh]
 optional = true

{experimaestro-1.7.0rc0 → experimaestro-1.7.0rc2}/src/experimaestro/experiments/cli.py

@@ -194,7 +194,11 @@ def experiments_cli(  # noqa: C901
             xp_file = Path(xp_file)
             if not python_path:
                 python_path.append(xp_file.parent)
-            logging.info("Using python path: %s", ", ".join(str(s) for s in python_path))
+            logging.info(
+                "Using python path: %s", ", ".join(str(s) for s in python_path)
+            )
+    else:
+        xp_file = Path(xp_file)
 
     assert (
         module_name or xp_file
@@ -269,11 +273,11 @@ def experiments_cli(  # noqa: C901
 
     # Define the workspace
     ws_env = find_workspace(workdir=workdir, workspace=workspace)
-    
+
     workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
-    
+
     # --- Runs the experiment
     with experiment(
         ws_env, configuration.id, host=host, port=port, run_mode=run_mode

{experimaestro-1.7.0rc0 → experimaestro-1.7.0rc2}/src/experimaestro/launcherfinder/registry.py

@@ -1,5 +1,6 @@
 # Configuration registers
 
+from contextlib import contextmanager
 from typing import ClassVar, Dict, Optional, Set, Type, Union
 
 from pathlib import Path
@@ -36,6 +37,16 @@ def load_yaml(schema, path: Path):
         )
 
 
+@contextmanager
+def ensure_enter(fp):
+    """Behaves as a resource, whether it is one or not"""
+    if hasattr(fp, "__enter__"):
+        with fp as _fp:
+            yield _fp
+    else:
+        yield fp
+
+
 class LauncherRegistry:
     INSTANCES: ClassVar[Dict[Path, "LauncherRegistry"]] = {}
     CURRENT_CONFIG_DIR: ClassVar[Optional[Path]] = None
@@ -78,7 +89,7 @@ class LauncherRegistry:
 
             from importlib import util
 
-            with launchers_py.__fspath__() as fp:
+            with ensure_enter(launchers_py.__fspath__()) as fp:
                 spec = util.spec_from_file_location("xpm_launchers_conf", fp)
                 module = util.module_from_spec(spec)
                 spec.loader.exec_module(module)

experimaestro-1.7.0rc2/src/experimaestro/notifications.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+import "google/protobuf/any.proto";
+
+
+message Envelope {
+    google.protobuf.Any payload = 1;
+}
+
+message LevelInformation {
+    int32 level = 1;
+
+    optional string desc = 2; // Optional in proto3 is handled by default unless you want to use 'optional'
+    float progress = 3;
+
+    float previous_progress = 4;
+    string previous_desc = 5;
+}

{experimaestro-1.7.0rc0 → experimaestro-1.7.0rc2}/src/experimaestro/notifications.py

@@ -12,15 +12,16 @@ from tqdm.auto import tqdm as std_tqdm
 
 from .utils import logger
 from experimaestro.taskglobals import Env as TaskEnv
+from experimaestro.notifications_pb2 import LevelInformation
 
 # --- Progress and other notifications
 
 T = TypeVar("T")
 
-
 @dataclass
 class LevelInformation:
     level: int
+    
     desc: Optional[str]
     progress: float
 
@@ -213,6 +214,7 @@ class Reporter(threading.Thread):
 
                 self.cv.notify_all()
 
+    #: The reporter instance
     INSTANCE: ClassVar[Optional["Reporter"]] = None
 
     @staticmethod

experimaestro-1.7.0rc2/src/experimaestro/notifications_pb2.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: src/experimaestro/notifications.proto
+# Protobuf Python Version: 5.29.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    5,
+    29,
+    3,
+    '',
+    'src/experimaestro/notifications.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%src/experimaestro/notifications.proto\x1a\x19google/protobuf/any.proto\"1\n\x08\x45nvelope\x12%\n\x07payload\x18\x01 \x01(\x0b\x32\x14.google.protobuf.Any\"\x81\x01\n\x10LevelInformation\x12\r\n\x05level\x18\x01 \x01(\x05\x12\x11\n\x04\x64\x65sc\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x08progress\x18\x03 \x01(\x02\x12\x19\n\x11previous_progress\x18\x04 \x01(\x02\x12\x15\n\rprevious_desc\x18\x05 \x01(\tB\x07\n\x05_descb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.experimaestro.notifications_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_ENVELOPE']._serialized_start=68
+  _globals['_ENVELOPE']._serialized_end=117
+  _globals['_LEVELINFORMATION']._serialized_start=120
+  _globals['_LEVELINFORMATION']._serialized_end=249
+# @@protoc_insertion_point(module_scope)

experimaestro-1.7.0rc2/src/experimaestro/streaming_state.py

@@ -0,0 +1,124 @@
+"""Handles streaming states
+
+Allows to track a global state via a file
+
+- partial states can be output
+- every K bytes, a global state is output
+
+The reader can then recover quickly by seeking the last global state output.
+
+"""
+
+import struct
+from io import BufferedWriter, BufferedReader, SEEK_END
+from typing import Type, Generator, Union
+
+from google.protobuf.message import Message
+
+# Constants
+PARTIAL_UPDATE = 1
+GLOBAL_STATE = 2
+HEADER_FORMAT = ">BI"  # 1-byte type tag, 4-byte message size
+HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
+MAGIC_NUMBER = b"\xab\xcd\xef\x00"
+MAGIC_SIZE = len(MAGIC_NUMBER)
+
+
+class StreamingStateWriter:
+    def __init__(
+        self,
+        file: BufferedWriter,
+        partial_cls: Type[Message],
+        snapshot_cls: Type[Message],
+        chunk_size: int = 8192,
+    ):
+        self.file = file
+        self.partial_cls = partial_cls
+        self.snapshot_cls = snapshot_cls
+        self.chunk_size = chunk_size
+        self.bytes_since_last_snapshot = 0
+
+    def write_partial(self, message: Message):
+        assert isinstance(message, self.partial_cls)
+        self._write_message(PARTIAL_UPDATE, message)
+
+    def write_snapshot_if_needed(self, message: Message):
+        assert isinstance(message, self.snapshot_cls)
+        if self.bytes_since_last_snapshot >= self.chunk_size:
+            self._write_message(GLOBAL_STATE, message, with_magic=True)
+            self.bytes_since_last_snapshot = 0
+
+    def _write_message(self, type_tag: int, message: Message, with_magic: bool = False):
+        data = message.SerializeToString()
+        header = struct.pack(HEADER_FORMAT, type_tag, len(data))
+        if with_magic:
+            self.file.write(MAGIC_NUMBER)
+            self.bytes_since_last_snapshot += MAGIC_SIZE
+        self.file.write(header + data)
+        self.file.flush()
+        self.bytes_since_last_snapshot += HEADER_SIZE + len(data)
+
+
+class StreamingStateReader:
+    def __init__(
+        self,
+        file: BufferedReader,
+        partial_cls: Type[Message],
+        snapshot_cls: Type[Message],
+    ):
+        self.file = file
+        self.partial_cls = partial_cls
+        self.snapshot_cls = snapshot_cls
+
+    def __iter__(self) -> Generator[Union[Message, Message], None, None]:
+        while True:
+            peek = self.file.peek(MAGIC_SIZE)[:MAGIC_SIZE]
+            if peek == MAGIC_NUMBER:
+                self.file.read(MAGIC_SIZE)
+
+            header = self.file.read(HEADER_SIZE)
+            if len(header) < HEADER_SIZE:
+                break
+
+            type_tag, length = struct.unpack(HEADER_FORMAT, header)
+            payload = self.file.read(length)
+
+            if type_tag == PARTIAL_UPDATE:
+                msg = self.partial_cls()
+            elif type_tag == GLOBAL_STATE:
+                msg = self.snapshot_cls()
+            else:
+                raise ValueError(f"Unknown type tag: {type_tag}")
+
+            msg.ParseFromString(payload)
+            yield msg
+
+    def seek_last_snapshot(self) -> Union[Message, None]:
+        self.file.seek(0, SEEK_END)
+        file_size = self.file.tell()
+
+        window = 4096
+        pos = file_size
+
+        while pos > 0:
+            read_size = min(window, pos)
+            pos -= read_size
+            self.file.seek(pos)
+            data = self.file.read(read_size)
+
+            idx = data.rfind(MAGIC_NUMBER)
+            if idx != -1:
+                snapshot_pos = pos + idx + MAGIC_SIZE
+                self.file.seek(snapshot_pos)
+                header = self.file.read(HEADER_SIZE)
+                if len(header) < HEADER_SIZE:
+                    break
+                type_tag, length = struct.unpack(HEADER_FORMAT, header)
+                if type_tag != GLOBAL_STATE:
+                    continue
+                payload = self.file.read(length)
+                msg = self.snapshot_cls()
+                msg.ParseFromString(payload)
+                return msg
+
+        return None

experimaestro-1.7.0rc2/src/experimaestro/tests/state_streamer.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+message PartialUpdate {
+    int32 updated_field = 1;
+}
+
+message GlobalState {
+    int32 field = 1;
+}

experimaestro-1.7.0rc2/src/experimaestro/tests/state_streamer_pb2.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: src/experimaestro/tests/state_streamer.proto
+# Protobuf Python Version: 5.29.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    5,
+    29,
+    3,
+    '',
+    'src/experimaestro/tests/state_streamer.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,src/experimaestro/tests/state_streamer.proto\"&\n\rPartialUpdate\x12\x15\n\rupdated_field\x18\x01 \x01(\x05\"\x1c\n\x0bGlobalState\x12\r\n\x05\x66ield\x18\x01 \x01(\x05\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.experimaestro.tests.state_streamer_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_PARTIALUPDATE']._serialized_start=48
+  _globals['_PARTIALUPDATE']._serialized_end=86
+  _globals['_GLOBALSTATE']._serialized_start=88
+  _globals['_GLOBALSTATE']._serialized_end=116
+# @@protoc_insertion_point(module_scope)

experimaestro-1.7.0rc2/src/experimaestro/tests/test_state_streamer.py

@@ -0,0 +1,18 @@
+from .state_streamer_pb2 import PartialUpdate, GlobalState
+
+# Writing
+with open("log.bin", "wb") as f:
+    writer = StreamingStateWriter(f, PartialUpdate, GlobalState)
+    for i in range(1000):
+        writer.write_partial(PartialUpdate(updated_field=i))
+        if i % 100 == 0:
+            writer.write_snapshot_if_needed(GlobalState(field=i))
+
+# Reading
+with open("log.bin", "rb") as f:
+    reader = StreamingStateReader(f, PartialUpdate, GlobalState)
+    last_snapshot = reader.seek_last_snapshot()
+    print("Last Snapshot:", last_snapshot)
+
+    for msg in reader:
+        print(type(msg).__name__, msg)