xopt 3.2.1__tar.gz → 3.2.2__tar.gz

{xopt-3.2.1 → xopt-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xopt
-Version: 3.2.1
+Version: 3.2.2
 Summary: Flexible optimization of arbitrary problems in Python.
 Author: Christopher Mayes, Ryan Roussel, Nikita Kuklev, Christopher Pierce, Sara Miskovich
 License:                                  Apache License

{xopt-3.2.1 → xopt-3.2.2}/environment.yml

@@ -15,6 +15,7 @@ dependencies:
   - tqdm
   - orjson
   - matplotlib
+  - gest-api>=0.2
 # parallel
   - mpi4py
   - dask
@@ -33,3 +34,4 @@ dependencies:
   - mkdocstrings-python
   - ruff
   - typing-extensions
+  - gest-api

{xopt-3.2.1 → xopt-3.2.2}/xopt/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '3.2.1'
-__version_tuple__ = version_tuple = (3, 2, 1)
+__version__ = version = '3.2.2'
+__version_tuple__ = version_tuple = (3, 2, 2)
 
-__commit_id__ = commit_id = 'g46cc86a6b'
+__commit_id__ = commit_id = 'gd4c488f75'

{xopt-3.2.1 → xopt-3.2.2}/xopt/base.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from copy import deepcopy
 from typing import Any, Optional, Union
 
@@ -40,6 +41,11 @@ from .errors import DataError
 
 logger = logging.getLogger(__name__)
 
+DUMP_FILE_RENAME_MESSAGE = (
+    "The attribute `dump_file` in `Xopt` has been renamed to `xopt_dump_file`. "
+    "Support for the old name will be removed in later versions of the Xopt library."
+)
+
 
 class Xopt(XoptBaseModel):
     """
@@ -59,9 +65,13 @@ class Xopt(XoptBaseModel):
     strict : bool, optional
         A flag indicating whether exceptions raised during evaluation should stop the
         optimization process.
-    dump_file : str, optional
+    xopt_dump_file : str, optional
         An optional file path for dumping attributes of the xopt object and the
-        results of evaluations.
+        results of evaluations. Environment variables and `~` are expanded when
+        dumping.
+    data_dump_file : str, optional
+        An optional file path for dumping the evaluation data as a CSV file.
+        Environment variables and `~` are expanded when dumping.
     data : DataFrame, optional
         An optional DataFrame object for storing internal data related to the optimization
         process.
@@ -98,6 +108,8 @@ class Xopt(XoptBaseModel):
         Serializes the Xopt configuration to a YAML string.
     dump(file: str = None, **kwargs)
         Dumps the Xopt configuration to a specified file.
+    dump_data()
+        Dumps the evaluation data to the file given by `data_dump_file`.
     dict(**kwargs) -> dict
         Provides a custom dictionary representation of the Xopt configuration.
     json(**kwargs) -> str
@@ -115,8 +127,11 @@ class Xopt(XoptBaseModel):
         description="flag to indicate if exceptions raised during evaluation "
         "should stop Xopt",
     )
-    dump_file: Optional[str] = Field(
-        None, description="file to dump the results of the evaluations"
+    xopt_dump_file: Optional[str] = Field(
+        None, description="file to dump the serialized Xopt object to"
+    )
+    data_dump_file: Optional[str] = Field(
+        None, description="file to dump the evaluation data to as CSV"
     )
     data: Optional[DataFrame] = Field(None, description="internal DataFrame object")
     serialize_torch: bool = Field(
@@ -243,6 +258,35 @@ class Xopt(XoptBaseModel):
             )
         return data
 
+    @model_validator(mode="before")
+    @classmethod
+    def dump_file_legacy(cls, data: Any):
+        """
+        Handle backward compatibility: convert the old dump_file parameter to
+        xopt_dump_file.
+        """
+        if isinstance(data, dict) and "dump_file" in data:
+            warnings.warn(DUMP_FILE_RENAME_MESSAGE, DeprecationWarning, stacklevel=2)
+            dump_file = data.pop("dump_file")
+            if dump_file is not None:
+                if data.get("xopt_dump_file") is not None:
+                    raise ValueError(
+                        "Cannot specify both 'dump_file' and 'xopt_dump_file'. "
+                        "Use 'xopt_dump_file' instead."
+                    )
+                data["xopt_dump_file"] = dump_file
+        return data
+
+    @property
+    def dump_file(self) -> Optional[str]:
+        warnings.warn(DUMP_FILE_RENAME_MESSAGE, DeprecationWarning, stacklevel=2)
+        return self.xopt_dump_file
+
+    @dump_file.setter
+    def dump_file(self, value: Optional[str]):
+        warnings.warn(DUMP_FILE_RENAME_MESSAGE, DeprecationWarning, stacklevel=2)
+        self.xopt_dump_file = value
+
     @property
     def n_data(self) -> int:
         if self.data is None:
@@ -440,9 +484,11 @@ class Xopt(XoptBaseModel):
 
         self.add_data(output_data)
 
-        # dump data to file if specified
-        if self.dump_file is not None:
+        # dump to file(s) if specified
+        if self.xopt_dump_file is not None:
             self.dump()
+        if self.data_dump_file is not None:
+            self.dump_data()
 
         return output_data
 
@@ -624,31 +670,58 @@ class Xopt(XoptBaseModel):
 
     def dump(self, file: str = None, **kwargs):
         """
-        Dump data to a file.
+        Dump the Xopt configuration and data to a YAML file.
+
+        Environment variables and `~` in the path are expanded.
 
         Parameters
         ----------
         file : str, optional
             The path to the file where the Xopt configuration will be dumped.
+            Defaults to the `xopt_dump_file` attribute.
         **kwargs
             Additional keyword arguments for customizing the dump.
 
         Raises
         ------
         ValueError
-            If no dump file is specified via argument or in the `dump_file` attribute.
+            If no dump file is specified via argument or in the `xopt_dump_file`
+            attribute.
 
         """
-        fname = file if file is not None else self.dump_file
+        fname = file if file is not None else self.xopt_dump_file
 
         if fname is None:
             raise ValueError(
-                "no dump file specified via argument or in `dump_file` attribute"
+                "no dump file specified via argument or in `xopt_dump_file` attribute"
             )
-        else:
-            with open(fname, "w") as f:
-                f.write(self.yaml(**kwargs))
-            logger.debug(f"Dumped state to YAML file: {fname}")
+
+        fname = os.path.expanduser(os.path.expandvars(fname))
+        with open(fname, "w") as f:
+            f.write(self.yaml(**kwargs))
+        logger.debug(f"Dumped state to YAML file: {fname}")
+
+    def dump_data(self):
+        """
+        Dump the evaluation data to the CSV file given by `data_dump_file`.
+
+        Environment variables and `~` in the path are expanded.
+
+        Raises
+        ------
+        ValueError
+            If no data dump file is specified in the `data_dump_file` attribute.
+
+        """
+        if self.data_dump_file is None:
+            raise ValueError(
+                "no data dump file specified in `data_dump_file` attribute"
+            )
+
+        fname = os.path.expanduser(os.path.expandvars(self.data_dump_file))
+        data = self.data if self.data is not None else pd.DataFrame()
+        data.to_csv(fname, index_label="xopt_index")
+        logger.debug(f"Dumped data to CSV file: {fname}")
 
     def dict(self, **kwargs) -> dict:
         """

{xopt-3.2.1 → xopt-3.2.2}/xopt/generators/bayesian/turbo.py

@@ -256,10 +256,11 @@ class TurboController(XoptBaseModel, ABC):
                     if active_indices:
                         lengthscales = lengthscales[active_indices]
 
-                    # calculate the ratios of lengthscales for each axis
+                    # Normalize by the geometric mean without forming the raw
+                    # product, which can overflow or underflow.
                     active_dim = len(active_variable_names)
-                    weights = lengthscales / torch.prod(lengthscales) ** (
-                        1 / active_dim
+                    weights = lengthscales / torch.prod(
+                        lengthscales ** (1 / active_dim)
                     )
 
             # calculate the tr bounding box

xopt-3.2.2/xopt/generators/checkpoints.py

@@ -0,0 +1,123 @@
+from datetime import datetime
+from pydantic import BaseModel, Field, model_validator
+import json
+import os
+
+from ..vocs import VOCS
+
+
+class CheckpointMixin(BaseModel):
+    """
+    Mix-in class adding checkpoint saving and loading to a generator.
+
+    Checkpoints are written to a caller-supplied directory. The VOCS object is
+    serialized into the checkpoint itself. Legacy checkpoints which predate this
+    instead carry the VOCS object in a "vocs.txt" file one level above the
+    directory holding the checkpoints and are still supported when loading.
+
+    The host class must provide a ``vocs`` attribute and a ``to_json`` method.
+    Writing of the checkpoint file is the responsibility of the concrete class
+    as it will be implementation dependent.
+
+    Parameters
+    ----------
+    checkpoint_file : str, optional
+        Path to checkpoint file to load from. If provided, the generator will be
+        initialized from the checkpoint state. User-specified parameters will
+        override checkpoint values.
+    """
+
+    checkpoint_file: str | None = Field(
+        None, description="Path to checkpoint file to load from", exclude=True
+    )
+
+    @staticmethod
+    def _load_checkpoint_data(fname: str) -> dict:
+        """
+        Internal function to load generator data from checkpoint file as well as VOCS object.
+
+        Parameters
+        ----------
+        fname : str
+            Path to the checkpoint file
+
+        Returns
+        -------
+        dict
+            Dictionary containing VOCS and checkpoint data
+        """
+        # Load the checkpoint
+        with open(fname) as f:
+            checkpoint_data = json.load(f)
+
+        if "vocs" in checkpoint_data:
+            return checkpoint_data
+
+        # Legacy checkpoints w/o VOCS
+        vocs_fname = os.path.join(os.path.dirname(fname), "../vocs.txt")
+        if not os.path.exists(vocs_fname):
+            raise ValueError(
+                f'Checkpoint "{fname}" does not contain a VOCS object and no '
+                f'VOCS file was found at "{vocs_fname}".'
+            )
+
+        with open(vocs_fname) as f:
+            vocs = VOCS(**json.load(f))
+
+        return {"vocs": vocs, **checkpoint_data}
+
+    @model_validator(mode="before")
+    @classmethod
+    def load_from_checkpoint(cls, values):
+        """
+        Load from checkpoint file if checkpoint_file is provided.
+        """
+        # Case when a checkpoint file has been supplied
+        if isinstance(values, dict) and "checkpoint_file" in values:
+            checkpoint_file = values.pop("checkpoint_file")
+            if checkpoint_file is not None:
+                # Load checkpoint data
+                checkpoint_data = cls._load_checkpoint_data(checkpoint_file)
+
+                # Merge with user data precedence
+                merged_data = {**checkpoint_data, **values}
+                return merged_data
+
+        # No checkpoint
+        return values
+
+    def _save_checkpoint(self, path: str | os.PathLike) -> str:
+        """
+        Write a checkpoint of the generator state to disk.
+
+        Parameters
+        ----------
+        path : str or os.PathLike
+            Directory into which the checkpoint file is written. Created if it
+            does not already exist.
+
+        Returns
+        -------
+        str
+            Path to the checkpoint file which was written.
+        """
+        # Set up the output directory
+        os.makedirs(path, exist_ok=True)
+
+        # Create a base filename
+        base_checkpoint_filename = datetime.now().strftime("%Y%m%d_%H%M%S")
+        checkpoint_path = os.path.join(path, f"{base_checkpoint_filename}_1.txt")
+
+        # Check if file exists and increment counter until we find a free filename
+        counter = 2
+        while os.path.exists(checkpoint_path):
+            checkpoint_path = os.path.join(
+                path, f"{base_checkpoint_filename}_{counter}.txt"
+            )
+            counter += 1
+
+        # Now we have a unique filename
+        with open(checkpoint_path, "w") as f:
+            f.write(self.to_json())
+
+        return checkpoint_path

xopt-3.2.2/xopt/generators/ga/base.py

@@ -0,0 +1,239 @@
+from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
+import logging
+import os
+import pandas as pd
+import time
+
+from ..checkpoints import CheckpointMixin
+from ..deduplicated import DeduplicatedGeneratorBase
+
+POPULATION_METADATA_COLUMNS = [
+    "xopt_generation",
+    "xopt_candidate_idx",
+    "xopt_runtime",
+    "xopt_error",
+]
+
+
+class _InstanceLogger(logging.Logger):
+    """
+    Logger owned by a single object. Used for per-object logging in generator.
+    """
+
+    def __reduce__(self):
+        return _make_instance_logger, (self.name, self.parent.name, self.level)
+
+
+def _make_instance_logger(name: str, parent_name: str, level: int) -> logging.Logger:
+    """
+    Build an unregistered logger which propagates to an existing named logger.
+
+    Parameters
+    ----------
+    name : str
+        Name recorded on emitted records.
+    parent_name : str
+        Name of the logger records propagate to.
+    level : int
+        Level of the new logger.
+
+    Returns
+    -------
+    logging.Logger
+    """
+    logger = _InstanceLogger(name, level)
+    logger.parent = logging.getLogger(parent_name)
+    return logger
+
+
+class _ResolvedOutputDirMixin(BaseModel):
+    """
+    This mixin just adds a validator to strip the parameter `output_dir_resolved` from input.
+    This is saved as a computed field by the  model, but will error on loading from checkpoint
+    due to extra fields being disallows. It has to be a mixin here to get run before the
+    `CheckpointMixin` is fired.
+    """
+
+    @model_validator(mode="before")
+    @classmethod
+    def drop_resolved_output_dir(cls, values):
+        """Discard "output_dir_resolved" so a reloaded generator resolves its own."""
+        if isinstance(values, dict) and "output_dir_resolved" in values:
+            values = dict(values)
+            values.pop("output_dir_resolved")
+        return values
+
+
+class GAGeneratorBase(
+    CheckpointMixin, _ResolvedOutputDirMixin, DeduplicatedGeneratorBase
+):
+    """
+    Base class for genetic algorithm generators which write output and checkpoints.
+
+    Handles the output directory, log file, and periodic checkpointing on behalf of
+    subclasses. Subclasses call `end_generation` once each time a generation is
+    completed and everything else is taken care of.
+
+    Nothing is written to disk until the generator is used, so building or
+    deserializing one never touches the filesystem. Each generator owns its logger.
+
+    Parameters
+    ----------
+    output_dir : str or os.PathLike, optional
+        Directory to save algorithm state and population history, or None to write
+        nothing. Stored as a string and never modified; environment variables and "~"
+        are expanded when the path is used. If the directory already contains data, a
+        number is appended to avoid overwriting it.
+    checkpoint_freq : int, default=1
+        Frequency (in generations) at which checkpoints are saved. Set to -1 to
+        disable checkpointing.
+    log_level : int
+        Level of log messages written to "log.txt".
+
+    Attributes
+    ----------
+    output_dir_resolved : str
+        Expanded, collision free path `output_dir` was resolved to. All file writes go
+        here. Empty until the directory has been created.
+    """
+
+    output_dir: str | None = Field(
+        None,
+        description="Directory to save algorithm state and population history, or None "
+        "to write nothing. Environment variables and a leading '~' are expanded when "
+        "it is used and a number is appended if it already contains data",
+    )
+    checkpoint_freq: int = Field(
+        1,
+        description="How often (in generations) to save checkpoints (set to -1 to disable)",
+    )
+    log_level: int = Field(
+        logging.INFO, description="Log message level output to log.txt"
+    )
+    _output_dir_resolved: str = ""  # Empty until the output directory has been created
+
+    @field_validator("output_dir", mode="before")
+    @classmethod
+    def validate_output_dir(cls, value):
+        """Accept any os.PathLike, storing it as a string."""
+        if isinstance(value, os.PathLike):
+            return os.fspath(value)
+        return value
+
+    @computed_field
+    @property
+    def output_dir_resolved(self) -> str:
+        """Directory output is written to, empty until it has been created."""
+        return self._output_dir_resolved
+
+    def model_post_init(self, context):
+        # Get a unique logger owned by this instance.
+        self._logger = _make_instance_logger(
+            f"{type(self).__module__}.{type(self).__name__}",
+            type(self).__module__,
+            self.log_level,
+        )
+
+    def _prepare_output(self) -> None:
+        """
+        Resolve and create the output directory and begin logging to file.
+
+        Repeated calls do nothing. The requested path is expanded and, if it already
+        holds data, a number is appended. The result is kept in `output_dir_resolved`.
+        """
+        if (self.output_dir is None) or self._output_dir_resolved:
+            return
+
+        # Check if directory exists and do collision avoidance. Suffixes are applied to
+        # the expanded path so that they cannot land inside an environment variable or
+        # "~". The path is normalized first so that a trailing separator does not put
+        # the suffixed directory inside the one being protected.
+        requested = os.path.normpath(
+            os.path.expanduser(os.path.expandvars(self.output_dir))
+        )
+        resolved = requested
+        counter = 2
+        while os.path.exists(resolved) and os.listdir(resolved):
+            resolved = f"{requested}_{counter}"
+            counter += 1
+        if resolved != requested:
+            self._logger.info(
+                f'detected existing output_dir "{requested}" and corrected '
+                f'to "{resolved}" to avoid overwriting'
+            )
+
+        # We are now setup
+        os.makedirs(resolved, exist_ok=True)
+        self._output_dir_resolved = resolved
+
+        # Set up file logging
+        log_file_path = os.path.join(resolved, "log.txt")
+        file_handler = logging.FileHandler(log_file_path, mode="w")
+        file_handler.setLevel(self.log_level)
+        file_handler.setFormatter(
+            logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        )
+        self._logger.addHandler(file_handler)
+        self._logger.info(f"routing log output to file: {log_file_path}")
+
+        # Record the problem definition alongside the data
+        # Note: this is necessary to include in output for users running analysis on the results
+        # ie to plot Pareto front, you need to know the names and direction of the objectives
+        with open(os.path.join(resolved, "vocs.txt"), "w") as f:
+            f.write(self.vocs.model_dump_json())
+
+    def end_generation(self, generation_index: int, population: list[dict]) -> None:
+        """
+        Record a completed generation, writing output and checkpoints as configured.
+
+        Parameters
+        ----------
+        generation_index : int
+            Index of the generation which was just completed.
+        population : list of dict
+            The individuals making up the completed population.
+        """
+        self._prepare_output()
+        if not self._output_dir_resolved:
+            return
+        output_dir = self._output_dir_resolved
+        save_start_t = time.perf_counter()
+
+        # Save all Xopt data
+        self.data.to_csv(os.path.join(output_dir, "data.csv"), index=False)
+
+        # Construct the DataFrame for this population
+        pop_df = pd.DataFrame(population)
+        pop_df["xopt_generation"] = generation_index
+
+        # Normalize the columns in the DataFrame
+        # Avoid schema changing part way through optimization so we can write CSV in append mode
+        pop_df = pop_df.reindex(
+            columns=self.vocs.all_names + POPULATION_METADATA_COLUMNS
+        )
+
+        # Write population DataFrame to file
+        csv_path = os.path.join(output_dir, "populations.csv")
+        pop_df.to_csv(
+            csv_path, index=False, mode="a", header=not os.path.isfile(csv_path)
+        )
+        self._logger.debug(
+            f'saved optimization data to "{output_dir}" '
+            f"in {1000 * (time.perf_counter() - save_start_t):.2f}ms"
+        )
+
+        # Save a checkpoint if one is due
+        if self.checkpoint_freq > 0 and (generation_index % self.checkpoint_freq == 0):
+            checkpoint_path = self._save_checkpoint(
+                os.path.join(output_dir, "checkpoints")
+            )
+            self._logger.debug(f'saved checkpoint file "{checkpoint_path}"')
+
+    def close_log_file(self):
+        """
+        Closes out the log file (if used)
+        """
+        for handler in list(self._logger.handlers):
+            if isinstance(handler, logging.FileHandler):
+                handler.close()
+            self._logger.removeHandler(handler)

{xopt-3.2.1 → xopt-3.2.2}/xopt/generators/ga/cnsga.py

@@ -2,7 +2,7 @@ import array
 import logging
 import os
 import random
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import pandas as pd
 from deap import algorithms as deap_algorithms, base as deap_base, tools as deap_tools
@@ -88,6 +88,8 @@ class CNSGAGenerator(Generator):
     )
     _children: List[Dict] = PrivateAttr([])
     _offspring: Optional[pd.DataFrame] = PrivateAttr(None)
+    _loaded_population: pd.DataFrame | None = PrivateAttr(None)
+    _toolbox: Any = PrivateAttr(None)
     population: Optional[pd.DataFrame] = Field(None)
 
     model_config = ConfigDict(extra="allow")