potassco-benchmark-tool 2.1.1__py3-none-any.whl → 2.2.0__py3-none-any.whl

benchmarktool/resultparser/clasp.py

@@ -32,25 +32,32 @@ PAR = 2
 
 # pylint: disable=unused-argument
 def parse(
-    root: str, runspec: "runscript.Runspec", instance: "runscript.Benchmark.Instance"
+    path: str, runspec: "runscript.Runspec", instance: "runscript.Benchmark.Instance", run: int
 ) -> dict[str, tuple[str, Any]]:
     """
     Extracts some clasp statistics.
 
     Attributes:
-        root (str):                    The folder with results.
+        path (str):                    The path to the run directory.
         runspec (Runspec):             The run specification of the benchmark.
         instance (Benchmark.Instance): The benchmark instance.
+        run (int):                     The run number.
     """
     timeout = runspec.project.job.timeout
     res: dict[str, tuple[str, Any]] = {"time": ("float", timeout)}
     for f in ["runsolver.solver", "runsolver.watcher"]:
-        with open(os.path.join(root, f), errors="ignore", encoding="utf-8") as file:
-            for line in file:
-                for val, reg in clasp_re.items():
-                    m = reg[1].match(line)
-                    if m:
-                        res[val] = (reg[0], float(m.group("val")) if reg[0] == "float" else m.group("val"))
+        try:
+            with open(os.path.join(path, f), errors="ignore", encoding="utf-8") as file:
+                for line in file:
+                    for val, reg in clasp_re.items():
+                        m = reg[1].match(line)
+                        if m:
+                            res[val] = (reg[0], float(m.group("val")) if reg[0] == "float" else m.group("val"))
+        except FileNotFoundError:
+            sys.stderr.write(
+                f"*** WARNING: Result file '{f}' not found for run {run} of instance '{instance.name}' "
+                f"for system '{runspec.system.name}-{runspec.system.version}'! ({path})\n"
+            )
 
     if "rstatus" in res and res["rstatus"][1] == "out of memory":
         res["error"] = ("string", "std::bad_alloc")
@@ -70,7 +77,11 @@ def parse(
     if timedout:
         res["time"] = ("float", timeout)
     if error:
-        sys.stderr.write("*** ERROR: Run {0} failed with unrecognized status or error!\n".format(root))
+        sys.stderr.write(
+            f"*** WARNING: Run {run} of instance '{instance.name}' "
+            f"for system '{runspec.system.name}-{runspec.system.version}' "
+            f"failed with unrecognized status or error! ({path})\n"
+        )
     result["error"] = ("float", int(error))
     result["timeout"] = ("float", int(timedout))
     result["memout"] = ("float", int(memout))

benchmarktool/runscript/parser.py

@@ -6,6 +6,7 @@ representation in form of python classes.
 
 __author__ = "Roland Kaminski"
 import os
+import sys
 from typing import Any
 
 from lxml import etree  # type: ignore[import-untyped]
@@ -112,13 +113,14 @@ class Parser:
                     </xs:sequence>
                     <xs:attribute name="name" type="nameType" use="required"/>
                     <xs:attribute name="cmdline" type="xs:string"/>
+                    <xs:attribute name="cmdline_post" type="xs:string"/>
                     <xs:attribute name="tag">
                         <xs:simpleType>
                             <xs:list itemType="nameType"/>
                         </xs:simpleType>
                     </xs:attribute>
-                    <xs:attribute name="disttemplate" type="xs:string"/>
-                    <xs:attribute name="distopts" type="xs:string"/>
+                    <xs:attribute name="dist_template" type="xs:string"/>
+                    <xs:attribute name="dist_options" type="xs:string"/>
                     <xs:anyAttribute processContents="lax"/>
                 </xs:complexType>
             </xs:element>
@@ -128,6 +130,7 @@ class Parser:
         <xs:attribute name="measures" type="nameType" use="required"/>
         <xs:attribute name="config" type="nameType" use="required"/>
         <xs:attribute name="cmdline" type="xs:string"/>
+        <xs:attribute name="cmdline_post" type="xs:string"/>
     </xs:complexType>
 
     <!-- generic attributes for jobs-->
@@ -136,6 +139,7 @@ class Parser:
         <xs:attribute name="timeout" type="timeType" use="required"/>
         <xs:attribute name="runs" type="xs:positiveInteger" use="required"/>
         <xs:attribute name="memout" type="xs:positiveInteger"/>
+        <xs:attribute name="template_options" type="xs:string"/>
         <xs:anyAttribute processContents="lax"/>
     </xs:attributeGroup>
 
@@ -326,145 +330,188 @@ class Parser:
         )
         schema = etree.XMLSchema(schemadoc)
 
-        with open(file_name, "r", encoding="utf8") as file:
-            doc = etree.parse(file)
-            schema.assertValid(doc)
-
-            root = doc.getroot()
-            run = Runscript(root.get("output"))
+        try:
+            doc = etree.parse(file_name)
+        except (etree.XMLSyntaxError, OSError) as e:
+            if isinstance(e, OSError):
+                sys.stderr.write(f"*** ERROR: Runscript file '{file_name}' not found.\n")
+                sys.exit(1)
+            sys.stderr.write(f"*** ERROR: XML Syntax Error in runscript: {e}\n")
+            sys.exit(1)
 
-            job: DistJob | SeqJob
-
-            for node in root.xpath("./distjob"):
+        try:
+            schema.assertValid(doc)
+        except etree.DocumentInvalid as e:
+            sys.stderr.write(f"*** ERROR: Invalid runscript file: {e}\n")
+            sys.exit(1)
+
+        root = doc.getroot()
+        run = Runscript(root.get("output"))
+
+        for node in root.xpath("./distjob"):
+            run.add_job(self._parse_job(node, "distjob"))
+
+        for node in root.xpath("./seqjob"):
+            run.add_job(self._parse_job(node, "seqjob"))
+
+        for node in root.xpath("./machine"):
+            machine = Machine(node.get("name"), node.get("cpu"), node.get("memory"))
+            run.add_machine(machine)
+
+        for node in root.xpath("./config"):
+            config = Config(node.get("name"), node.get("template"))
+            run.add_config(config)
+
+        compound_settings: dict[str, list[str]] = {}
+        system_order = 0
+        for node in root.xpath("./system"):
+            config = run.configs[node.get("config")]
+            system = System(node.get("name"), node.get("version"), node.get("measures"), system_order, config)
+            setting_order = 0
+            sys_cmdline = node.get("cmdline")
+            sys_cmdline_post = node.get("cmdline_post")
+            for child in node.xpath("setting"):
                 attr = self._filter_attr(
-                    node, ["name", "timeout", "runs", "script_mode", "walltime", "cpt", "partition"]
+                    child, ["name", "cmdline", "cmdline_post", "tag", "dist_options", "dist_template"]
                 )
-
-                partition = node.get("partition")
-                if partition is None:
-                    partition = "kr"
-                job = DistJob(
-                    node.get("name"),
-                    tools.xml_to_seconds_time(node.get("timeout")),
-                    int(node.get("runs")),
-                    attr,
-                    node.get("script_mode"),
-                    tools.xml_to_seconds_time(node.get("walltime")),
-                    int(node.get("cpt")),
-                    partition,
+                compound_settings[child.get("name")] = []
+                dist_template = child.get("dist_template")
+                if dist_template is None:
+                    dist_template = "templates/single.dist"
+                if child.get("tag") is None:
+                    tag = set()
+                else:
+                    tag = set(child.get("tag").split(None))
+                dist_options = child.get("dist_options")
+                if dist_options is None:
+                    dist_options = ""
+                encodings: dict[str, set[str]] = {"_default_": set()}
+                for grandchild in child.xpath("./encoding"):
+                    if grandchild.get("enctag") is None:
+                        encodings["_default_"].add(os.path.normpath(grandchild.get("file")))
+                    else:
+                        enctags = set(grandchild.get("enctag").split(None))
+                        for t in enctags:
+                            if t not in encodings:
+                                encodings[t] = set()
+                            encodings[t].add(os.path.normpath(grandchild.get("file")))
+
+                cmdline = " ".join(
+                    filter(None, [sys_cmdline, child.get("cmdline"), sys_cmdline_post, child.get("cmdline_post")])
                 )
-                run.add_job(job)
-
-            for node in root.xpath("./seqjob"):
-                attr = self._filter_attr(node, ["name", "timeout", "runs", "parallel"])
-                job = SeqJob(
-                    node.get("name"),
-                    tools.xml_to_seconds_time(node.get("timeout")),
-                    int(node.get("runs")),
-                    attr,
-                    int(node.get("parallel")),
+                name = child.get("name")
+                compound_settings[child.get("name")].append(name)
+                keys = list(attr.keys())
+                if keys:
+                    sys.stderr.write(
+                        f"""*** INFO: Attribute{'s' if len(keys) > 1 else ''} {', '.join(f"'{k}'" for k in keys)} in setting '{name}' {'are' if len(keys) > 1 else 'is'} currently unused.\n"""
+                    )
+                setting = Setting(
+                    name=name,
+                    cmdline=cmdline,
+                    tag=tag,
+                    order=setting_order,
+                    dist_template=dist_template,
+                    attr=attr,
+                    dist_options=dist_options,
+                    encodings=encodings,
                 )
-                run.add_job(job)
-
-            for node in root.xpath("./machine"):
-                machine = Machine(node.get("name"), node.get("cpu"), node.get("memory"))
-                run.add_machine(machine)
-
-            for node in root.xpath("./config"):
-                config = Config(node.get("name"), node.get("template"))
-                run.add_config(config)
-
-            compound_settings: dict[str, list[str]] = {}
-            system_order = 0
-            for node in root.xpath("./system"):
-                config = run.configs[node.get("config")]
-                system = System(node.get("name"), node.get("version"), node.get("measures"), system_order, config)
-                setting_order = 0
-                sys_cmdline = node.get("cmdline")
-                for child in node.xpath("setting"):
-                    attr = self._filter_attr(child, ["name", "cmdline", "tag", "distopts", "disttemplate"])
-                    compound_settings[child.get("name")] = []
-                    disttemplate = child.get("disttemplate")
-                    if disttemplate is None:
-                        disttemplate = "templates/single.dist"
-                    if child.get("tag") is None:
-                        tag = set()
-                    else:
-                        tag = set(child.get("tag").split(None))
-                    dist_options = child.get("distopts")
-                    if dist_options is None:
-                        dist_options = ""
-                    encodings: dict[str, set[str]] = {"_default_": set()}
-                    for grandchild in child.xpath("./encoding"):
-                        if grandchild.get("enctag") is None:
-                            encodings["_default_"].add(os.path.normpath(grandchild.get("file")))
-                        else:
-                            enctags = set(grandchild.get("enctag").split(None))
-                            for t in enctags:
-                                if t not in encodings:
-                                    encodings[t] = set()
-                                encodings[t].add(os.path.normpath(grandchild.get("file")))
-
-                    cmdline = " ".join(filter(None, [sys_cmdline, child.get("cmdline")]))
-                    name = child.get("name")
-                    compound_settings[child.get("name")].append(name)
-                    setting = Setting(name, cmdline, tag, setting_order, disttemplate, attr, dist_options, encodings)
-                    system.add_setting(setting)
-                    setting_order += 1
-
-                run.systems[(system.name, system.version)] = system
-                system_order += 1
-
-            element: Any
-            for node in root.xpath("./benchmark"):
-                benchmark = Benchmark(node.get("name"))
-                for child in node.xpath("./folder"):
-                    if child.get("group") is not None:
-                        group = child.get("group").lower() == "true"
-                    else:
-                        group = False
-                    element = Benchmark.Folder(child.get("path"), group)
-                    if child.get("enctag") is None:
-                        tag = set()
-                    else:
-                        tag = set(child.get("enctag").split(None))
-                    element.add_enctags(tag)
-                    for grandchild in child.xpath("./encoding"):
-                        element.add_encoding(grandchild.get("file"))
-                    for grandchild in child.xpath("./ignore"):
-                        element.add_ignore(grandchild.get("prefix"))
-                    benchmark.add_element(element)
-                for child in node.xpath("./files"):
-                    element = Benchmark.Files(child.get("path"))
-                    if child.get("enctag") is None:
-                        tag = set()
-                    else:
-                        tag = set(child.get("enctag").split(None))
-                    element.add_enctags(tag)
-                    for grandchild in child.xpath("./encoding"):
-                        element.add_encoding(grandchild.get("file"))
-                    for grandchild in child.xpath("./add"):
-                        element.add_file(grandchild.get("file"), grandchild.get("group"))
-                    benchmark.add_element(element)
-                run.add_benchmark(benchmark)
-
-            for node in root.xpath("./project"):
-                project = Project(node.get("name"), run, run.jobs[node.get("job")])
-                run.add_project(project)
-                for child in node.xpath("./runspec"):
-                    for setting_name in compound_settings[child.get("setting")]:
-                        project.add_runspec(
-                            child.get("machine"),
-                            child.get("system"),
-                            child.get("version"),
-                            setting_name,
-                            child.get("benchmark"),
-                        )
-
-                for child in node.xpath("./runtag"):
-                    project.add_runtag(child.get("machine"), child.get("benchmark"), child.get("tag"))
+                system.add_setting(setting)
+                setting_order += 1
+
+            run.systems[(system.name, system.version)] = system
+            system_order += 1
+
+        element: Any
+        for node in root.xpath("./benchmark"):
+            benchmark = Benchmark(node.get("name"))
+            for child in node.xpath("./folder"):
+                if child.get("group") is not None:
+                    group = child.get("group").lower() == "true"
+                else:
+                    group = False
+                element = Benchmark.Folder(child.get("path"), group)
+                if child.get("enctag") is None:
+                    tag = set()
+                else:
+                    tag = set(child.get("enctag").split(None))
+                element.add_enctags(tag)
+                for grandchild in child.xpath("./encoding"):
+                    element.add_encoding(grandchild.get("file"))
+                for grandchild in child.xpath("./ignore"):
+                    element.add_ignore(grandchild.get("prefix"))
+                benchmark.add_element(element)
+            for child in node.xpath("./files"):
+                element = Benchmark.Files(child.get("path"))
+                if child.get("enctag") is None:
+                    tag = set()
+                else:
+                    tag = set(child.get("enctag").split(None))
+                element.add_enctags(tag)
+                for grandchild in child.xpath("./encoding"):
+                    element.add_encoding(grandchild.get("file"))
+                for grandchild in child.xpath("./add"):
+                    element.add_file(grandchild.get("file"), grandchild.get("group"))
+                benchmark.add_element(element)
+            run.add_benchmark(benchmark)
+
+        for node in root.xpath("./project"):
+            project = Project(node.get("name"), run, run.jobs[node.get("job")])
+            run.add_project(project)
+            for child in node.xpath("./runspec"):
+                for setting_name in compound_settings[child.get("setting")]:
+                    project.add_runspec(
+                        machine_name=child.get("machine"),
+                        system_name=child.get("system"),
+                        system_version=child.get("version"),
+                        setting_name=setting_name,
+                        benchmark_name=child.get("benchmark"),
+                    )
+
+            for child in node.xpath("./runtag"):
+                project.add_runtag(child.get("machine"), child.get("benchmark"), child.get("tag"))
+
+        self.validate_components(run)
         return run
 
+    def validate_components(self, run: Runscript) -> None:
+        """
+        Check runscript for the existence of all required components.
+        """
+        # machine
+        if not run.machines:
+            sys.stderr.write("*** WARNING: No machine defined in runscript.\n")
+
+        # config
+        if not run.configs:
+            sys.stderr.write("*** WARNING: No config defined in runscript.\n")
+
+        # system
+        if not run.systems:
+            sys.stderr.write("*** WARNING: No system defined in runscript.\n")
+
+        # setting
+        for system in run.systems.values():
+            if not system.settings:
+                sys.stderr.write(f"*** WARNING: No setting defined for system '{system.name}-{system.version}'.\n")
+
+        # job
+        if not run.jobs:
+            sys.stderr.write("*** WARNING: No job defined in runscript.\n")
+
+        # benchmark
+        if not run.benchmarks:
+            sys.stderr.write("*** WARNING: No benchmark defined in runscript.\n")
+
+        # instances
+        for benchmark in run.benchmarks.values():
+            if not benchmark.elements:
+                sys.stderr.write(f"*** WARNING: No instance folder/files defined for benchmark '{benchmark.name}'.\n")
+
+        # project
+        if not run.projects:
+            sys.stderr.write("*** WARNING: No project defined in runscript.\n")
+
     def _filter_attr(self, node: etree._Element, skip: list[str]) -> dict[str, Any]:
         """
         Returns a dictionary containing all attributes of a given node.
@@ -475,3 +522,57 @@ class Parser:
             if not key in skip:
                 attr[key] = val
         return attr
+
+    def _parse_job(self, node: etree._Element, job_type: str) -> DistJob | SeqJob:
+        """
+        Parses a job node and returns the corresponding job instance.
+        """
+        attr_filter = ["name", "timeout", "memout", "runs", "template_options"]
+        kwargs = {
+            "name": node.get("name"),
+            "timeout": tools.xml_to_seconds_time(node.get("timeout")),
+            "runs": int(node.get("runs")),
+        }
+        memout = node.get("memout")
+        if memout is not None:
+            kwargs["memout"] = int(memout)
+        template_options = node.get("template_options")
+        if template_options is None:
+            template_options = ""
+        kwargs["template_options"] = template_options
+
+        if job_type == "distjob":
+            attr = self._filter_attr(node, attr_filter + ["script_mode", "walltime", "cpt", "partition"])
+            keys = list(attr.keys())
+            if keys:
+                sys.stderr.write(
+                    f"""*** INFO: Attribute{'s' if len(keys) > 1 else ''} {', '.join(f"'{k}'" for k in keys)} in distjob '{node.get('name')}' {'are' if len(keys) > 1 else 'is'} currently unused.\n"""
+                )
+            kwargs.update(
+                {
+                    "attr": attr,
+                    "script_mode": node.get("script_mode"),
+                    "walltime": tools.xml_to_seconds_time(node.get("walltime")),
+                    "cpt": int(node.get("cpt")),
+                }
+            )
+            partition = node.get("partition")
+            if partition is not None:
+                kwargs["partition"] = partition
+            return DistJob(**kwargs)  # pylint: disable=missing-kwoa
+        if job_type == "seqjob":
+            attr = self._filter_attr(node, attr_filter + ["parallel"])
+            keys = list(attr.keys())
+            if keys:
+                sys.stderr.write(
+                    f"""*** INFO: Attribute{'s' if len(keys) > 1 else ''} {', '.join(f"'{k}'" for k in keys)} in seqjob '{node.get('name')}' {'are' if len(keys) > 1 else 'is'} currently unused.\n"""
+                )
+            kwargs.update(
+                {
+                    "attr": attr,
+                    "parallel": int(node.get("parallel")),
+                }
+            )
+            return SeqJob(**kwargs)  # pylint: disable=missing-kwoa
+        # Should never happen, checked by xml schema
+        raise ValueError(f"Unknown job type: {job_type}")  # nocoverage