potassco-benchmark-tool 2.1.1__py3-none-any.whl → 2.2.1__py3-none-any.whl

benchmarktool/entry_points.py

@@ -35,37 +35,41 @@ def btool_conv(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
     def run(args: Any) -> None:
         p = ResParser()
         if args.resultfile:
-            with open(args.resultfile, encoding="utf-8") as in_file:
-                res = p.parse(in_file)
+            try:
+                with open(args.resultfile, encoding="utf-8") as in_file:
+                    res = p.parse(in_file)
+            except FileNotFoundError:
+                sys.stderr.write(f"*** ERROR: Result file '{args.resultfile}' not found.\n")
+                sys.exit(1)
         else:
             res = p.parse(sys.stdin)
         export: bool = args.export
         if args.jupyter_notebook is not None:
             export = True
-        ex_file = res.gen_office(args.output, args.projects, args.measures, export)
+        ex_file = res.gen_spreadsheet(args.output, args.projects, args.measures, export, args.max_col_width)
         if args.jupyter_notebook is not None and ex_file is not None:
             gen_ipynb(ex_file, args.jupyter_notebook)
 
     def parse_set(s: str) -> set[str]:
         return set(filter(None, (x.strip() for x in s.split(","))))
 
-    def parse_measures(s: str) -> list[tuple[str, str | None]]:
-        measures = []
+    def parse_measures(s: str) -> dict[str, Any]:
+        measures = {}
         if s != "all":  # empty list = select all measures
             for x in s.split(","):
                 parts = x.split(":", 1)
                 if not parts[0]:
                     raise ArgumentTypeError(f"Invalid measure: '{x}'")
-                measures.append((parts[0], parts[1] if len(parts) > 1 else None))
+                measures[parts[0]] = parts[1] if len(parts) > 1 else None
         return measures
 
     conv_parser = subparsers.add_parser(
         "conv",
-        help="Convert results to ODS or other formats",
+        help="Convert results to XLSX or other formats",
         description=dedent(
             """\
-            Convert previously collected benchmark results to ODS file
-            and optionally generate Jupyter notebook.
+            Convert previously collected benchmark results to XLSX
+            spreadsheet and optionally generate Jupyter notebook.
             """
         ),
         formatter_class=formatter,
@@ -76,7 +80,20 @@ def btool_conv(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
 
     conv_parser.add_argument("resultfile", nargs="?", type=str, help="Result file (default: stdin)")
     conv_parser.add_argument(
-        "-o", "--output", default="out.ods", help="Name of generated ods file (default: out.ods)", metavar="<file.ods>"
+        "-o",
+        "--output",
+        type=str,
+        default="out.xlsx",
+        help="Name of generated xlsx file (default: %(default)s)",
+        metavar="<file.xlsx>",
+    )
+    conv_parser.add_argument(
+        "--max-col-width",
+        type=int,
+        default=300,
+        help="Maximum column width for spreadsheet (default: %(default)d)",
+        metavar="<n>",
+        dest="max_col_width",
     )
     conv_parser.add_argument(
         "-p",
@@ -96,7 +113,7 @@ def btool_conv(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
             Measures to display
             Comma separated list of form 'name[:{t,to,-}]' (optional argument determines coloring)
             Use '-m all' to display all measures
-            (default: time:t,timeout:to)
+            (default: %(default)s)
             """
         ),
         metavar="<measure[:{t,to,-}][,measure[:{t,to,-}],...]>",
@@ -105,13 +122,12 @@ def btool_conv(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
         "-e",
         "--export",
         action="store_true",
-        help="Export instance data to parquet file (same name as ods file)",
+        help="Export instance data to parquet file (same name as .xlsx file)",
     )
     conv_parser.add_argument(
         "-j",
         "--jupyter-notebook",
         type=str,
-        nargs="?",
         help=dedent(
             """\
             Name of generated .ipynb file
@@ -146,7 +162,7 @@ def btool_eval(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
         type=int,
         default=2,
         dest="par_x",
-        help="Add penalized-average-runtime score factor as measure (default: 2)",
+        help="Add penalized-average-runtime score factor as measure (default: %(default)d)",
         metavar="<n>",
     )
     eval_parser.set_defaults(func=run)
@@ -160,7 +176,7 @@ def btool_gen(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
     def run(args: Any) -> None:
         p = RunParser()
         run = p.parse(args.runscript)
-        run.gen_scripts(args.exclude)
+        run.gen_scripts(args.exclude, args.force)
 
     gen_parser = subparsers.add_parser(
         "gen",
@@ -170,6 +186,12 @@ def btool_gen(subparsers: "_SubParsersAction[ArgumentParser]") -> None:
     )
     gen_parser.add_argument("runscript", type=str, help="Runscript file", metavar="<runscript.xml>")
     gen_parser.add_argument("-e", "--exclude", action="store_true", help="Exclude finished runs")
+    gen_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Overwrite existing files",
+    )
     gen_parser.set_defaults(func=run)
 
 
@@ -178,7 +200,7 @@ def btool_init(subparsers: "_SubParsersAction[ArgumentParser]") -> None:  # noco
     Register init subcommand.
     """
 
-    def copy_dir(src_dir: str, dst_dir: str, overwrite: bool = False) -> None:
+    def copy_dir(src_dir: str, dst_dir: str, force: bool = False) -> None:
         """
         Copy directory src_dir to dst_dir.
         By default existing files are not overwritten.
@@ -199,33 +221,36 @@ def btool_init(subparsers: "_SubParsersAction[ArgumentParser]") -> None:  # noco
                 if not os.path.isdir(target_dir):
                     os.mkdir(target_dir)
                 else:
-                    sys.stderr.write(f"INFO: Directory already exists:\t{target_dir}\n")
+                    sys.stderr.write(f"*** INFO: Directory already exists:\t{target_dir}\n")
             # Files
             for file in files:
                 source_name = os.path.join(root, file)
                 target_name = os.path.join(target_root, file)
                 if os.path.isfile(target_name):
-                    sys.stderr.write(f"INFO: File already exists:\t{target_name}\n")
-                    if not overwrite:
+                    sys.stderr.write(f"*** INFO: File already exists:\t{target_name}\n")
+                    if not force:
                         continue
                 shutil.copy(source_name, target_name)
 
     def run(args: Any) -> None:
         src_dir = os.path.join(os.path.dirname(__file__), "init")
         if not os.path.isdir(src_dir):
-            raise SystemExit(f"Resources missing: '{src_dir}' does not exist.\nTry reinstalling the package.")
+            sys.stderr.write(
+                f"*** ERROR: Resources missing: '{src_dir}' does not exist.\nTry reinstalling the package.\n"
+            )
+            sys.exit(1)
         cwd = os.getcwd()
-        copy_dir(src_dir, cwd, args.overwrite)
+        copy_dir(src_dir, cwd, args.force)
         rp_dir = os.path.join(cwd, "resultparsers")
         if not os.path.isdir(rp_dir):
             os.mkdir(rp_dir)
         else:
-            sys.stderr.write(f"INFO: Directory already exists:\t{rp_dir}\n")
+            sys.stderr.write(f"*** INFO: Directory already exists:\t{rp_dir}\n")
         if args.resultparser_template:
             rp_tmp = os.path.join(rp_dir, "rp_tmp.py")
             if os.path.isfile(rp_tmp):
-                sys.stderr.write(f"INFO: File already exists:\t{rp_tmp}\n")
-                if not args.overwrite:
+                sys.stderr.write(f"*** INFO: File already exists:\t{rp_tmp}\n")
+                if not args.force:
                     return
             shutil.copy(os.path.join(os.path.dirname(__file__), "resultparser", "clasp.py"), rp_tmp)
 
@@ -236,14 +261,14 @@ def btool_init(subparsers: "_SubParsersAction[ArgumentParser]") -> None:  # noco
             """\
             Initialize the benchmark environment with the necessary directory structure
             and example runscript and templates.
-            By default existing files are not overwritten; use --overwrite to change this behavior.
+            By default existing files are not overwritten; use --force to change this behavior.
             """
         ),
         formatter_class=formatter,
     )
     parser.add_argument(
-        "-o",
-        "--overwrite",
+        "-f",
+        "--force",
         action="store_true",
         help="Overwrite existing files",
     )
@@ -270,9 +295,15 @@ def btool_run_dist(subparsers: "_SubParsersAction[ArgumentParser]") -> None:  #
         return len([f for f in result.stdout.strip().splitlines() if f])
 
     def run(args: Any) -> None:
-        pending = [
-            f for f in os.listdir(args.folder) if os.path.isfile(os.path.join(args.folder, f)) and f.endswith(".dist")
-        ]
+        try:
+            pending = [
+                f
+                for f in os.listdir(args.folder)
+                if os.path.isfile(os.path.join(args.folder, f)) and f.endswith(".dist")
+            ]
+        except FileNotFoundError:
+            sys.stderr.write(f"*** ERROR: Folder '{args.folder}' not found.\n")
+            sys.exit(1)
         print(f"Found {len(pending)} jobs to dispatch.")
         while pending:
             jobs = running_jobs(args.user)
@@ -311,17 +342,17 @@ def btool_run_dist(subparsers: "_SubParsersAction[ArgumentParser]") -> None:  #
     parser.add_argument(
         "-j",
         "--jobs",
-        help="Maximum number of jobs running at once (default: 100)",
         type=int,
         default=100,
+        help="Maximum number of jobs running at once (default: %(default)d)",
         metavar="<n>",
     )
     parser.add_argument(
         "-w",
         "--wait",
-        help="Time to wait between checks in seconds (default: 1)",
         type=int,
         default=1,
+        help="Time to wait between checks in seconds (default: %(default)d)",
         metavar="<n>",
     )
     parser.set_defaults(func=run)
@@ -340,15 +371,22 @@ def btool_verify(subparsers: Any) -> None:  # nocoverage
             for file in files:
                 if file == "runsolver.watcher":
                     watcher_path = os.path.join(root, file)
+                    if os.path.getsize(watcher_path) == 0:
+                        sys.stderr.write(f"*** WARNING: Empty watcher file: {watcher_path}\n")
+                        continue
                     with open(watcher_path, encoding="utf-8") as f:
                         if "runlim error" in f.read():
                             error_files.append(watcher_path)
+                elif file == "runsolver.solver":
+                    solver_path = os.path.join(root, file)
+                    if os.path.getsize(solver_path) == 0:
+                        sys.stderr.write(f"*** WARNING: Empty solver file: {solver_path}\n")
         return error_files
 
     def run(args: Any) -> None:
         folder = args.folder
         if not os.path.isdir(folder):
-            print("Error: provided folder doesn't exist", file=sys.stderr)
+            sys.stderr.write(f"*** ERROR: Folder '{folder}' not found.\n")
             sys.exit(1)
 
         if error_files := find_runlim_errors(folder):

benchmarktool/init/runscripts/runscript-all.xml

@@ -16,12 +16,12 @@
 
 	<config name="dist-generic" template="templates/seq-generic.sh"/>
 	<system name="clingo" version="5.8.0" measures="clasp" config="dist-generic">
-		<setting name="one-as" tag="one-as" cmdline="--stats 1" disttemplate="templates/single.dist" distopts="#SBATCH --hint=compute_bound"/>
+		<setting name="one-as" tag="one-as" cmdline="--stats 1" dist_template="templates/single.dist" dist_options="#SBATCH --hint=compute_bound"/>
 		<setting name="all-as" tag="all-as" cmdline="--stats -q 0"/>
 	</system>
 
 	<seqjob name="seq-generic" timeout="120s" runs="1" memout="1000" parallel="8"/>
-	<distjob name="dist-generic" timeout="120s" runs="1" memout="1000" script_mode="timeout" walltime="23h 59m 59s" cpt="4" partition="short"/>
+	<distjob name="dist-generic" timeout="120s" runs="1" memout="1000" template_options="--single" script_mode="timeout" walltime="23h 59m 59s" cpt="4" partition="short"/>
 
 	<benchmark name="seq-suite">
 		<folder path="benchmarks/clasp" group="true">

benchmarktool/init/runscripts/runscript-dist.xml

@@ -4,10 +4,10 @@
 
 	<config name="dist-generic" template="templates/seq-generic.sh"/>
 	<system name="clasp" version="3.4.0" measures="clasp" config="dist-generic">
-		<setting name="one-as" tag="one-as" cmdline="--stats 1" disttemplate="templates/single.dist"/>
+		<setting name="one-as" tag="one-as" cmdline="--stats 1" dist_template="templates/single.dist"/>
 	</system>
 
-	<distjob name="dist-generic" timeout="1200s" runs="1" script_mode="timeout" walltime="23h 59m 59s" cpt="4"/>
+	<distjob name="dist-generic" timeout="1200s" runs="1" template_options="--single" script_mode="timeout" walltime="23h 59m 59s" cpt="4"/>
 
 	<benchmark name="dist-suite">
 		<folder path="benchmarks/clasp"/>

benchmarktool/init/runscripts/runscript-example.xml

@@ -12,7 +12,7 @@
 
 	<seqjob name="seq-gen" timeout="900s" runs="1" parallel="1"/>
 
-	<distjob name="dist-gen" timeout="1200s" runs="1" script_mode="timeout" walltime="23h 59m 59s" cpt="4"/>
+	<distjob name="dist-gen" timeout="1200s" runs="1" template_options="--single" script_mode="timeout" walltime="23h 59m 59s" cpt="4"/>
 
 	<benchmark name="no-pigeons">
 		<folder path="benchmarks/clasp/">

benchmarktool/init/templates/seq-generic.sh

@@ -1,12 +1,27 @@
 #!/bin/bash
 # https://github.com/arminbiere/runlim
 
+CAT="{root}/programs/gcat.sh"
+
 cd "$(dirname $0)"
 
-[[ -e .finished ]] || "{run.root}/programs/runlim" \
-	--space-limit={run.memout} \
-	--output-file=runsolver.watcher \
-	--real-time-limit={run.timeout} \
-	"{run.root}/programs/{run.solver}" {run.args} {run.files} {run.encodings} > runsolver.solver
+runner=( "{root}/programs/runlim" \
+  {options} \
+  --space-limit={memout} \
+  --output-file=runsolver.watcher \
+  --real-time-limit={timeout} \
+  "{root}/programs/{solver}" {args})
+
+input=( {files} {encodings} )
+
+if [[ ! -e .finished ]]; then
+  {{
+    if file -b --mime-type -L  "${{input[@]}}" | grep -qv "text/"; then
+      "$CAT" "${{input[@]}}" | "${{runner[@]}}"
+    else
+      "${{runner[@]}}" "${{input[@]}}"
+    fi
+  }} > runsolver.solver
+fi
 
 touch .finished

benchmarktool/result/ipynb_gen.py

@@ -464,6 +464,8 @@ display(out_plot)
         nbf.v4.new_code_cell(plot_code),
     ]
     fname = file_name
+    if not fname.lower().endswith(".ipynb"):
+        fname += ".ipynb"
     nb.cells[1]["metadata"]["jp-MarkdownHeadingCollapsed"] = True
     nb.cells[3]["metadata"]["jp-MarkdownHeadingCollapsed"] = True
     # nb.cells[6]["metadata"]["jupyter"] = {"source_hidden": True}

benchmarktool/result/result.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Iterator, Optional
 
-from benchmarktool.result.ods_gen import ODSDoc
+from benchmarktool.result.xlsx_gen import XLSXDoc
 
 
 class Result:
@@ -45,17 +45,25 @@ class Result:
                 benchmarks.add(runspec.benchmark)
         return BenchmarkMerge(benchmarks)
 
-    def gen_office(
-        self, out: str, sel_projects: set[str], measures: list[tuple[str, Any]], export: bool = False
+    # pylint: disable=too-many-positional-arguments
+    def gen_spreadsheet(
+        self,
+        out: str,
+        sel_projects: set[str],
+        measures: dict[str, Any],
+        export: bool = False,
+        max_col_width: int = 300,
     ) -> Optional[str]:
         """
-        Prints the current result in open office spreadsheet format.
+        Prints the current result in Microsoft Excel Spreadsheet format (XLSX).
         Returns the name of the export file if values are exported.
 
         Attributes:
             out (str):                        The output file to write to.
             sel_projects (set[str]):          The selected projects ("" for all).
-            measures (list[tuple[str, Any]]): The measures to extract.
+            measures (dict[str, Any]):        The measures to extract.
+            export (bool):                    Whether to export the raw values as parquet file.
+            max_col_width (int):              The maximum column width for spreadsheet.
         """
         projects: list[Project] = []
         for project in self.projects.values():
@@ -63,16 +71,18 @@ class Result:
                 projects.append(project)
         benchmark_merge = self.merge(projects)
 
-        sheet = ODSDoc(benchmark_merge, measures)
+        doc = XLSXDoc(benchmark_merge, measures, max_col_width)
         for project in projects:
             for runspec in project:
-                sheet.add_runspec(runspec)
-        sheet.finish()
-        sheet.make_ods(out)
+                doc.add_runspec(runspec)
+        doc.finish()
+        if not out.lower().endswith(".xlsx"):
+            out += ".xlsx"
+        doc.make_xlsx(out)
 
         if export:
             # as_posix() for windows compatibility
-            ex_file = Path(out).absolute().as_posix().replace(".ods", ".parquet")
+            ex_file = Path(out).absolute().as_posix().replace(".xlsx", ".parquet")
             timeout_meta = {}
             for project in projects:
                 for runspec in project.runspecs:
@@ -84,7 +94,7 @@ class Result:
                         + "/"
                         + runspec.setting.name
                     ] = [self.jobs[project.job].timeout]
-            sheet.inst_sheet.export_values(ex_file, timeout_meta)
+            doc.inst_sheet.export_values(ex_file, timeout_meta)
             return ex_file
         return None
 
@@ -397,7 +407,7 @@ class ClassResult:
         yield from self.instresults
 
 
-@dataclass(order=True, frozen=True)
+@dataclass(order=True, frozen=True, eq=True)
 class InstanceResult:
     """
     Represents the result of an individual instance (with possibly multiple runs).
@@ -432,7 +442,7 @@ class Run:
     number: int
     measures: dict[str, tuple[str, str]] = field(default_factory=dict, compare=False)
 
-    def iter(self, measures: list[tuple[str, Any]]) -> Iterator[tuple[str, str, str]]:
+    def iter(self, measures: dict[str, Any]) -> Iterator[tuple[str, str, str]]:
         """
         Creates an iterator over all measures captured during the run.
         Measures can be filter by giving a string set of measure names.
@@ -440,13 +450,13 @@ class Run:
         will be returned.
 
         Attributes:
-            measures (list[tuple[str, Any]]): Selected measures.
+            measures (dict[str, Any]): Selected measures.
         """
-        if len(measures) == 0:
+        if len(measures.keys()) == 0:
             for name in sorted(self.measures.keys()):
                 yield name, self.measures[name][0], self.measures[name][1]
         else:
-            for name, _ in measures:
+            for name in measures.keys():
                 if name in self.measures:
                     yield name, self.measures[name][0], self.measures[name][1]
                 else: