iqm-benchmarks 1.3__py3-none-any.whl

iqm/benchmarks/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2024 IQM Benchmarks developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+IQM's Python Library Benchmarking Suite QCVV.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .benchmark_definition import AnalysisResult, Benchmark, RunResult
+
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = "iqm-benchmarks"
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError

iqm/benchmarks/benchmark.py

@@ -0,0 +1,109 @@
+# Copyright 2024 IQM Benchmarks developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generic Benchmark class
+"""
+
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from typing import Dict, Literal, Optional, OrderedDict, Type
+
+from matplotlib.figure import Figure
+from pydantic import BaseModel
+
+from iqm.qiskit_iqm.iqm_backend import IQMBackendBase
+
+
+class BenchmarkBase(ABC):
+    """
+    The base implementation of all benchmarks, from which they inherit.
+    """
+
+    def __init__(
+        self,
+        backend: IQMBackendBase,
+        configuration: "BenchmarkConfigurationBase",
+    ):
+        """Construct the BenchmarkBase class.
+
+        Args:
+            backend (IQMBackendBase): the backend to execute the benchmark on
+            configuration (BenchmarkConfigurationBase): the configuration of the benchmark
+        """
+        self.backend = backend
+        self.configuration = configuration
+        self.serializable_configuration = deepcopy(self.configuration)
+        self.serializable_configuration.benchmark = self.name()
+
+        self.shots = self.configuration.shots
+        self.calset_id = self.configuration.calset_id
+        self.max_gates_per_batch = self.configuration.max_gates_per_batch
+
+        self.routing_method = self.configuration.routing_method
+        self.physical_layout = self.configuration.physical_layout
+
+        self.raw_data: Dict = {}
+        self.job_meta: Dict = {}
+        self.results: Dict = {}
+        self.raw_results: Dict = {}
+
+        self.untranspiled_circuits: Dict[str, Dict[int, list]] = {}
+        self.transpiled_circuits: Dict[str, Dict[int, list]] = {}
+
+        self.figures: Dict[str, Figure] = {}
+
+    @classmethod
+    @abstractmethod
+    def name(cls):
+        """Return the name of the benchmark."""
+
+    @abstractmethod
+    def execute_full_benchmark(self):
+        """Execute the full benchmark on the given backend."""
+
+    @staticmethod
+    def check_requirements(all_benchmarks: OrderedDict[str, "BenchmarkBase"]) -> OrderedDict[str, "BenchmarkBase"]:
+        """Check whether the requirements for the benchmark are met, returning a valid benchmark dictionary."""
+        return all_benchmarks
+
+    def generate_requirements(self, all_benchmarks: OrderedDict[str, "BenchmarkBase"]) -> None:
+        """Generate the required attributes for execution."""
+
+
+class BenchmarkConfigurationBase(BaseModel):
+    """Benchmark configuration base.
+
+    Attributes:
+        benchmark (Type[BenchmarkBase]): the benchmark configuration.
+        shots (int): the number of shots to use in circuit execution.
+                * Default for all benchmarks is 2**8.
+        max_gates_per_batch (Optional[int]): the maximum number of gates per circuit batch.
+                * Default for all benchmarks is None.
+        calset_id (Optional[str]): the calibration ID to use in circuit execution.
+                * Default for all benchmarks is None (uses last available calibration ID).
+        routing_method (Literal["basic", "lookahead", "stochastic", "sabre", "none"]): the Qiskit routing method to use in transpilation.
+                * Default for all benchmarks is "sabre".
+        physical_layout (Literal["fixed", "batching"]): whether physical layout is constrained during transpilation to selected physical qubits.
+                - "fixed": physical layout is constrained during transpilation to the selected initial physical qubits.
+                - "batching": physical layout is allowed to use any other physical qubits, and circuits are batched according to final measured qubits.
+                * Default for all benchmarks is "fixed".
+    """
+
+    benchmark: Type[BenchmarkBase]
+    shots: int = 2**8
+    max_gates_per_batch: Optional[int] = None
+    calset_id: Optional[str] = None
+    routing_method: Literal["basic", "lookahead", "stochastic", "sabre", "none"] = "sabre"
+    physical_layout: Literal["fixed", "batching"] = "fixed"

iqm/benchmarks/benchmark_definition.py

@@ -0,0 +1,264 @@
+# Copyright 2024 IQM Benchmarks developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module implements the base class for defining a benchmark.
+"""
+
+from abc import ABC, abstractmethod
+import copy
+from copy import deepcopy
+from dataclasses import dataclass, field
+import functools
+from typing import Any, Dict, List, Union
+import uuid
+
+from matplotlib.figure import Figure
+import matplotlib.pyplot as plt
+import xarray as xr
+
+from iqm.benchmarks.benchmark import BenchmarkConfigurationBase
+from iqm.benchmarks.utils import get_iqm_backend, timeit
+from iqm.qiskit_iqm.iqm_backend import IQMBackendBase
+from iqm.qiskit_iqm.iqm_provider import IQMBackend, IQMFacadeBackend
+
+
+@dataclass
+class RunResult:
+    """
+    A dataclass that stores the results of a single run of a Benchmark.
+
+    RunResult should contain enough information that the Benchmark can be analyzed based on those
+    results.
+    """
+
+    dataset: xr.Dataset
+
+
+@dataclass
+class AnalysisResult:
+    """
+    A dataclass storing the results of the analysis.
+
+    The result consists of a dataset, plots, and observations. Plots are defined as a dictionary
+    that maps a plot name to a figure. Observations are key-value pairs of data that contain the
+    main results of the benchmark.
+    """
+
+    dataset: xr.Dataset
+    plots: dict[str, Figure] = field(default_factory=lambda: ({}))
+    observations: dict[str, Any] = field(default_factory=lambda: ({}))
+
+    def plot(self, plot_name: str):
+        """
+        Plots the given figure.
+
+        Args:
+            plot_name: Name of the figure to be plotted.
+        """
+        return self.plots[plot_name]
+
+    def plot_all(self):
+        """
+        Plots all the figures defined for the analysis.
+        """
+        for fig in self.plots.values():
+            show_figure(fig)
+            plt.show()
+
+    @classmethod
+    def from_run_result(cls, run: RunResult):
+        """
+        Creates a new ``AnalysisResult`` from a ``RunResult``.
+
+        Args:
+            run: A run for which analysis result is created.
+        """
+        return cls(dataset=run.dataset)
+
+
+def default_analysis_function(result: AnalysisResult) -> AnalysisResult:
+    """
+    The default analysis that only pass the result through.
+    """
+    return result
+
+
+def merge_datasets_dac(datasets: List[xr.Dataset]) -> xr.Dataset:
+    """
+    Merges a list of datasets recursively to minimize dataset sizes during merge
+    Args:
+        datasets: List[xr.Dateset]
+            A list of xarray datasets
+    Returns:
+        A list containing a single merged dataset
+    """
+    if len(datasets) == 1:
+        return datasets[0]
+    datasets_new = []
+    for i in range(0, len(datasets), 2):
+        if i == (len(datasets) - 1):
+            datasets_new.append(datasets[i])
+        else:
+            datasets_new.append(xr.merge(datasets[i : i + 2]))
+    return merge_datasets_dac(datasets_new)
+
+
+@timeit
+def add_counts_to_dataset(counts: List[Dict[str, int]], identifier: str, dataset: xr.Dataset):
+    """Adds the counts from a cortex job result to the given dataset.
+    If counts with the same identifier are already present in the old dataset, then both counts are added together.
+
+    Args:
+        counts (List[Dict[str, int]]): A list of dictionaries with counts of bitstrings.
+        identifier (str): A string to identify the current data, for instance the qubit layout.
+        dataset (xr.Dataset): Dataset to add results to.
+    Returns:
+        dataset_merged: xarray.Dataset
+            A merged dataset where the new counts are added the input dataset
+    """
+    if not isinstance(counts, list):
+        counts = [counts]
+    datasets = []
+    for ii, _ in enumerate(counts):
+        ds_temp = xr.Dataset()
+        counts_dict = dict(counts[ii])
+        counts_array = xr.DataArray(list(counts_dict.values()), {f"{identifier}_state_{ii}": list(counts_dict.keys())})
+        if f"{identifier}_counts_{ii}" in dataset.variables:
+            counts_array = counts_array + dataset[f"{identifier}_counts_{ii}"]
+        counts_array = counts_array.sortby(counts_array[f"{identifier}_state_{ii}"])
+        ds_temp.update({f"{identifier}_counts_{ii}": counts_array})
+        datasets.append(ds_temp)
+    dataset_new = merge_datasets_dac(datasets)
+    dataset_merged = dataset.merge(dataset_new, compat="override")
+    return dataset_merged
+
+
+def show_figure(fig):
+    """
+    Shows a closed figure.
+
+    Args:
+        fig: Figure to show.
+    """
+    dummy = plt.figure()
+    new_manager = dummy.canvas.manager
+    new_manager.canvas.figure = fig
+    fig.set_canvas(new_manager.canvas)
+
+
+class Benchmark(ABC):
+    """
+    A base class for running cortex-based Benchmark experiments.
+
+    In order to write a new benchmark, it is recommended to derive a class
+    from this baseclass. A new benchmark is defined by deriving from this base
+    class and implementing the execute method. Additionally, a custom analysis
+    for the benchmark can be implemented by giving the pointer to the analysis
+    method in ``analysis_function`` field. The given analysis_function should
+    accept ``AnalysisResult`` as its input and return the final result.
+    """
+
+    analysis_function = staticmethod(default_analysis_function)
+    default_options: dict[str, Any] | None = None
+    options: dict[str, Any] | None = None
+    # name: str = "unnamed_benchmark"
+
+    def __init__(self, backend: Union[str, IQMBackendBase], configuration: "BenchmarkConfigurationBase", **kwargs):
+
+        # Ported from BenchmarkBase   # CHECK
+        self.configuration = configuration
+        self.serializable_configuration = deepcopy(self.configuration)
+        self.serializable_configuration.benchmark = self.name
+
+        if isinstance(backend, str):
+            self.backend = get_iqm_backend(backend)
+        else:
+            assert isinstance(backend, IQMBackendBase)
+            self.backend = backend
+
+        self.shots = self.configuration.shots
+        self.calset_id = self.configuration.calset_id
+        self.max_gates_per_batch = self.configuration.max_gates_per_batch
+
+        self.routing_method = self.configuration.routing_method
+        self.physical_layout = self.configuration.physical_layout
+
+        self.untranspiled_circuits: Dict[str, Dict[int, list]] = {}
+        self.transpiled_circuits: Dict[str, Dict[int, list]] = {}
+
+        # From exa_support MR
+        self.options = copy.copy(self.default_options) if self.default_options else {}
+        self.options.update(kwargs)
+        self.runs: list[RunResult] = []
+
+    @classmethod
+    @abstractmethod
+    def name(cls):
+        """Return the name of the benchmark."""
+
+    @abstractmethod
+    def execute(self, backend: IQMBackend | IQMFacadeBackend | str) -> xr.Dataset:
+        """
+        Executes the benchmark.
+
+        This method should be overridden by deriving classes.
+
+        Args:
+            backend: Qiskit backend used to execute benchmarks.
+
+        Returns:
+            An xarray dataset which contains to results of the benchmark execution.
+            The dataset should contain all the information necessary for analysing
+            the benchmark results.
+        """
+
+    def run(self, calibration_set_id: str | uuid.UUID | None = None) -> RunResult:
+        """
+        Runs the benchmark using the given backend and calibration_set_id.
+
+        Args:
+            calibration_set_id:
+                CalibrationSetId used to initialize the backend or None for the latest calibration set.
+
+        Returns:
+            RunResult: The result of the benchmark run.
+        """
+        backend_for_execute = copy.copy(self.backend)
+        backend_for_execute.run = functools.partial(self.backend.run, calibration_set_id=calibration_set_id)  # type: ignore
+        dataset = self.execute(backend_for_execute)
+        run = RunResult(dataset)
+        self.runs.append(run)
+        return run
+
+    def analyze(self, run_index=-1) -> AnalysisResult:
+        """
+        The default analysis for the benchmark.
+
+        Internally uses the function defined by the attribute ``analysis_function``
+        to perform the analysis. This function makes a shallow copy of the dataset produced by
+        run. Therefore, it is recommended to not make changes to the data of the dataset but
+        just the structure of the array.
+
+        Args:
+            run_index: Index for the run to analyze.
+
+        Returns:
+            An analysis result constructed from the run and updated by the analysis method defined by
+            the ``analysis_function`` field.
+        """
+        run = self.runs[run_index]
+        result = AnalysisResult.from_run_result(run)
+        updated_result = self.analysis_function(result)
+        return updated_result

iqm/benchmarks/benchmark_experiment.py

@@ -0,0 +1,163 @@
+# Copyright 2024 IQM Benchmarks developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generic Benchmark Experiment class
+"""
+
+from copy import deepcopy
+from json import dump
+from pathlib import Path
+import pickle
+from time import strftime
+from typing import List, Optional, OrderedDict, Union
+
+from iqm.benchmarks.benchmark import BenchmarkBase, BenchmarkConfigurationBase
+from iqm.benchmarks.utils import get_iqm_backend
+from iqm.qiskit_iqm.iqm_backend import IQMBackendBase
+
+
+class BenchmarkExperiment:
+    """
+    A Benchmark Experiment wraps the execution of one or more benchmarks, checks their requirements are met, stores the
+    execution results and plots relevant figures.
+    """
+
+    def __init__(
+        self,
+        backend: Union[str, IQMBackendBase],
+        benchmark_configurations: List[BenchmarkConfigurationBase],
+        device_id: Optional[str] = None,
+    ):
+        """Construct the BenchmarkExperiment class.
+
+        Args:
+            backend (str | IQMBackendBase): the backend to execute the benchmarks on
+            benchmark_configurations (List[BenchmarkConfigurationBase]): the configuration(s) of the benchmark(s)
+            device_id (Optional[str], optional): the identifier of the device. Defaults to None.
+
+        Raises:
+            ValueError: backend not supported. Try 'garnet' or 'iqmfakeadonis'
+        """
+        self.timestamp = strftime("%Y%m%d-%H%M%S")
+
+        self.device_id = device_id if device_id is not None else backend
+
+        if isinstance(backend, str):
+            self.backend = get_iqm_backend(backend)
+
+        else:
+            assert isinstance(backend, IQMBackendBase)
+            self.backend = backend
+
+        benchmarks: OrderedDict[str, BenchmarkBase] = OrderedDict(
+            (config.benchmark.name(), config.benchmark(self.backend, config)) for config in benchmark_configurations
+        )
+
+        for benchmark in benchmarks.values():
+            benchmarks_copy = deepcopy(benchmarks)
+            benchmarks_copy = benchmark.check_requirements(benchmarks_copy)
+        self.benchmarks = benchmarks_copy
+
+    def run_experiment(self) -> None:
+        """Run the Benchmark experiment, and store the configuration, raw data, results and figures."""
+
+        for name, benchmark in self.benchmarks.items():
+            print("\nNow executing " + name)
+            # Create the directory for results
+            results_dir = f"Outputs/{self.device_id}/{self.timestamp}/{name}/"
+            Path(results_dir).mkdir(parents=True, exist_ok=True)
+
+            # Execute the current benchmark
+            benchmark.generate_requirements(self.benchmarks)
+            benchmark.execute_full_benchmark()
+
+            # Create configuration JSON file
+            with open(
+                f"{results_dir}{self.device_id}_{self.timestamp}_{name}_configuration.json",
+                "w",
+                encoding="utf-8",
+            ) as f_configuration:
+                dump(benchmark.serializable_configuration.model_dump(), f_configuration)
+
+            # Create untranspiled circuit files
+            if benchmark.untranspiled_circuits:
+                for key_qubits in benchmark.untranspiled_circuits.keys():
+                    for key_depth in benchmark.untranspiled_circuits[key_qubits].keys():
+                        with open(
+                            f"{results_dir}{self.device_id}_{self.timestamp}_{name}_qubits_{key_qubits}_depth_{key_depth}_untranspiled.pkl",
+                            "wb",
+                        ) as f_circuits:
+                            pickle.dump(
+                                benchmark.untranspiled_circuits[key_qubits][key_depth],
+                                f_circuits,
+                            )
+
+            # Create transpiled circuit files
+            for key_qubits in benchmark.transpiled_circuits.keys():
+                for key_depth in benchmark.transpiled_circuits[key_qubits].keys():
+                    with open(
+                        f"{results_dir}{self.device_id}_{self.timestamp}_{name}_qubits_{key_qubits}_depth_{key_depth}_transpiled.pkl",
+                        "wb",
+                    ) as f_circuits:
+                        pickle.dump(
+                            benchmark.transpiled_circuits[key_qubits][key_depth],
+                            f_circuits,
+                        )
+
+            # Create raw result pickle files
+            with open(
+                f"{results_dir}{self.device_id}_{self.timestamp}_{name}_raw_results.pkl",
+                "wb",
+            ) as f_raw_results:
+                pickle.dump(
+                    benchmark.raw_results,
+                    f_raw_results,
+                )
+
+            # Create raw data JSON file
+            with open(
+                f"{results_dir}{self.device_id}_{self.timestamp}_{name}_raw_data.json",
+                "w",
+                encoding="utf-8",
+            ) as f_raw_data:
+                dump(benchmark.raw_data, f_raw_data)
+
+            # Create job metadata JSON file
+            with open(
+                f"{results_dir}{self.device_id}_{self.timestamp}_{name}_job_metadata.json",
+                "w",
+                encoding="utf-8",
+            ) as f_job_metadata:
+                dump(benchmark.job_meta, f_job_metadata)
+
+            # Create results JSON file
+            with open(
+                f"{results_dir}{self.device_id}_{self.timestamp}_{name}_results.json",
+                "w",
+                encoding="utf-8",
+            ) as f_results:
+                dump(benchmark.results, f_results)
+
+            # Create figures
+            Path(f"{results_dir}figures/").mkdir(parents=True, exist_ok=True)
+            for fig_name, fig in benchmark.figures.items():
+                fig.savefig(
+                    f"{results_dir}figures/{self.device_id}_{self.timestamp}_{name}_{fig_name}",
+                    dpi=250,
+                    bbox_inches="tight",
+                )
+
+            # Save benchmark
+            self.benchmarks[benchmark.name()] = benchmark

iqm/benchmarks/compressive_gst/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 IQM Benchmarks developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Compressive GST is a self-consistent characterization method which constructs the process matrices for a set of gates,
+as well as full parametrizations of an initial state and a POVM.
+Low rank compression of the process matrix is used to reduce measurement and post-processing overhead.
+"""
+
+from . import compressive_gst