hector-cli 0.1.0__py3-none-any.whl

hector/core.py

@@ -0,0 +1,259 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Core plumbing: the extension Registry, shared constants, host<->container path
+translation, YAML/interpolation/matrix helpers, repl emitters, and the endpoint parser.
+
+You rarely need to touch this file. The interesting, extensible parts live in:
+  hubs.py       - add a cross-machine connection medium (uart, can, gpio, ...)
+  runners.py    - add a test runner (bash, robot, renode-test, ...)
+  mappings.py   - add a peripheral->host backend (file, ...)
+  modules.py    - add a module build kind (renode-verilator, ...)
+"""
+
+import itertools
+import os
+import re
+import subprocess
+from dataclasses import dataclass, field
+
+import yaml
+
+
+# ==========================================================================
+# REGISTRY  (the extension mechanism used across the codebase)
+# ==========================================================================
+class Registry:
+    """A named lookup table you extend with a decorator.
+
+        WIDGETS = Registry("widget")
+
+        @WIDGETS.register("foo")
+        class Foo: ...
+
+        WIDGETS.get("foo")          # -> Foo
+        "foo" in WIDGETS            # -> True
+        list(WIDGETS.keys())        # -> ["foo", ...]
+
+    Unknown keys raise a helpful error listing what *is* available, so a typo in
+    the YAML produces a clear message instead of a KeyError.
+    """
+    def __init__(self, name):
+        self.name = name
+        self._items = {}
+
+    def register(self, key):
+        def deco(obj):
+            if key in self._items:
+                raise ValueError(f"{self.name} '{key}' is already registered.")
+            self._items[key] = obj
+            return obj
+        return deco
+
+    def get(self, key):
+        if key not in self._items:
+            raise KeyError(
+                f"Unknown {self.name} '{key}'. Available: {', '.join(sorted(self._items)) or '(none)'}."
+            )
+        return self._items[key]
+
+    def __contains__(self, key):
+        return key in self._items
+
+    def keys(self):
+        return self._items.keys()
+
+
+# ==========================================================================
+# CONSTANTS
+# ==========================================================================
+TOOL_VERSION = "0.1.0"
+SUPPORTED_SCHEMA_VERSIONS = {"0.1"}
+
+RENODE_REPO_URL = "https://github.com/renode/renode.git"
+INTEGRATION_REPO_URL = "https://github.com/antmicro/renode-verilator-integration.git"
+
+ARTIFACTS_DIR = ".hector"          # everything we generate/clone lives here (gitignore it)
+WORKSPACE_MOUNT = os.getcwd()      # cwd is bind-mounted at the same path inside the container
+NO_DOCKER = False                  # when True, call renode/renode-test directly on host
+EXTRA_MOUNTS = []                  # dep dirs outside the workspace, bind-mounted 1:1 into every container
+LIBOPENLIBM_NAME = "libopenlibm-Linux-x86_64.a"
+
+DEFAULT_QUANTUM = "0.00001"        # 10 us; applied when any hub is present
+
+
+# ==========================================================================
+# RUNTIME OPTIONS  (execution-environment needs declared by components)
+# ==========================================================================
+@dataclass
+class RuntimeOptions:
+    """Execution-environment requirements a component (mapping / hub / module) needs
+    from the Renode run. Components declare these at parse time; the pipeline merges
+    them across all components and the docker layer maps them to `docker run` flags at
+    execution time. Add fields here as new needs arise (ports, mounts, ...)."""
+    capabilities: set = field(default_factory=set)   # -> docker --cap-add
+    devices: set = field(default_factory=set)         # -> docker --device
+    root: bool = False                                # -> docker -u 0:0 (run as root)
+
+    def merge(self, other):
+        """Fold another RuntimeOptions (or None) into this one."""
+        if other:
+            self.capabilities |= other.capabilities
+            self.devices |= other.devices
+            self.root = self.root or other.root
+        return self
+
+
+# ==========================================================================
+# PATH TRANSLATION  (host <-> container)
+# ==========================================================================
+def to_container_path(host_path):
+    abs_path = os.path.realpath(host_path)
+    root = os.path.realpath(os.getcwd())
+    if os.path.commonpath([abs_path, root]) != root:
+        raise ValueError(
+            f"Path '{host_path}' resolves outside the project root '{root}'. Everything used "
+            f"at runtime must live under the project folder (mounted at '{WORKSPACE_MOUNT}')."
+        )
+    return os.path.join(WORKSPACE_MOUNT, os.path.relpath(abs_path, root))
+
+
+def register_dependency_mount(host_dir):
+    """Return the container path for a dependency checkout (Renode source / integration),
+    bind-mounting it 1:1 into every container if it lives OUTSIDE the workspace.
+
+    Dirs under the workspace are already covered by the workspace mount, so they map via
+    to_container_path. An out-of-tree dir (e.g. a shared cache) is registered in
+    EXTRA_MOUNTS and used at its own absolute path inside the container."""
+    abs_dir = os.path.realpath(host_dir)
+    root = os.path.realpath(os.getcwd())
+    if os.path.commonpath([abs_dir, root]) == root:
+        return to_container_path(host_dir)
+    if abs_dir not in EXTRA_MOUNTS:
+        EXTRA_MOUNTS.append(abs_dir)
+    return abs_dir
+
+
+def container_path_for(host_path):
+    """Container path for a host path that may live under an out-of-tree dependency mount
+    (EXTRA_MOUNTS, bind-mounted 1:1); otherwise the normal workspace mapping."""
+    abs_p = os.path.realpath(host_path)
+    for m in EXTRA_MOUNTS:
+        m_abs = os.path.realpath(m)
+        if abs_p == m_abs or abs_p.startswith(m_abs + os.sep):
+            return abs_p
+    return to_container_path(host_path)
+
+
+def container_to_host(container_path):
+    if container_path != WORKSPACE_MOUNT and not container_path.startswith(WORKSPACE_MOUNT + os.sep):
+        raise ValueError(f"'{container_path}' is not under {WORKSPACE_MOUNT}.")
+    rel = os.path.relpath(container_path, WORKSPACE_MOUNT)
+    host = os.path.realpath(os.path.join(os.getcwd(), rel))
+    root = os.path.realpath(os.getcwd())
+    if os.path.commonpath([host, root]) != root:
+        raise ValueError(f"'{container_path}' maps outside the project root.")
+    return host
+
+
+def split_host_container(path):
+    if path == WORKSPACE_MOUNT or path.startswith(WORKSPACE_MOUNT + os.sep):
+        return container_to_host(path), path
+    host = os.path.realpath(path)
+    return host, to_container_path(host)
+
+
+# ==========================================================================
+# YAML / TEXT HELPERS
+# ==========================================================================
+def load_yaml(filepath):
+    with open(filepath, "r") as f:
+        return yaml.safe_load(f)
+
+
+def run_shell(cmd, cwd=None, shell="/bin/bash"):
+    subprocess.run(cmd, shell=True, check=True, executable=shell, cwd=cwd)
+
+
+def as_lines(section):
+    """Normalize a YAML block-string or list into meaningful lines (drop blanks and
+    full-line '#' comments)."""
+    if not section:
+        return []
+    raw = section.splitlines() if isinstance(section, str) else list(section)
+    return [s.strip() for s in (str(x) for x in raw)
+            if s.strip() and not s.strip().startswith("#")]
+
+
+# ==========================================================================
+# MATRIX & INTERPOLATION
+# ==========================================================================
+def expand_matrix(matrix_block):
+    if not matrix_block or "variables" not in matrix_block:
+        return [{}]
+    variables = matrix_block["variables"]
+    excludes = matrix_block.get("exclude", [])
+    all_jobs = [dict(zip(variables.keys(), combo))
+                for combo in itertools.product(*variables.values())]
+    return [job for job in all_jobs
+            if not any(all(job.get(k) == v for k, v in rule.items()) for rule in excludes)]
+
+
+def resolve_arguments(arguments, overrides=None):
+    """name -> value.  Precedence: --set flag > env var > config default."""
+    overrides = overrides or {}
+    resolved = {}
+    for name, default in (arguments or {}).items():
+        if name in overrides:
+            resolved[name] = overrides[name]
+            source = "cli"
+        elif name in os.environ:
+            resolved[name] = os.environ[name]
+            source = "env"
+        else:
+            resolved[name] = default
+            source = "default"
+        print(f"[ARG] {name} = {resolved[name]}  ({source})")
+    # --set can also introduce keys not declared in arguments:
+    for name, value in overrides.items():
+        if name not in resolved:
+            resolved[name] = value
+            print(f"[ARG] {name} = {value}  (cli, undeclared)")
+    return resolved
+
+
+def interpolate(data, context):
+    if isinstance(data, str):
+        return re.sub(r"\$\{([^}]+)\}",
+                      lambda m: str(context.get(m.group(1), m.group(0))), data)
+    if isinstance(data, dict):
+        return {k: interpolate(v, context) for k, v in data.items()}
+    if isinstance(data, list):
+        return [interpolate(item, context) for item in data]
+    return data
+
+
+# ==========================================================================
+# ENDPOINT PARSER  (connections + mappings)
+#   node.name        -> kind='plain'
+#   node.name@pin    -> kind='pin', pin=N   (Renode-style '@' pin / line number)
+#   name             -> kind='self'         (bare; typically a hub reference)
+# ==========================================================================
+def parse_endpoint(token):
+    token = token.strip()
+    if "." not in token:
+        return {"node": token, "name": token, "kind": "self", "raw": token}
+    node, rest = token.split(".", 1)
+    ep = {"node": node, "raw": rest}
+    m = re.match(r"^([A-Za-z0-9_]+)@(0x[0-9A-Fa-f]+|\d+)$", rest)
+    if m:
+        ep.update(kind="pin", name=m.group(1), pin=int(m.group(2), 0))
+        return ep
+    if re.match(r"^[A-Za-z0-9_]+$", rest):
+        ep.update(kind="plain", name=rest)
+        return ep
+    # node.periph.signal  (e.g. mcu.nvic.IRQ)
+    m2 = re.match(r"^([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)$", rest)
+    if m2:
+        ep.update(kind="signal", periph=m2.group(1), name=m2.group(1), signal=m2.group(2))
+        return ep
+    raise ValueError(f"Could not parse endpoint '{token}'")

hector/dependencies.py

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Managed clones of Renode and the verilator integration, pinned to renode_version."""
+
+import os
+
+from .core import (ARTIFACTS_DIR, INTEGRATION_REPO_URL, RENODE_REPO_URL,
+                   register_dependency_mount, run_shell)
+
+
+def prepare_dependencies(git_tag, renode_dir=None, integration_dir=None):
+    """Ensure the Renode source and verilator-integration checkouts exist, returning
+    their paths. Each location defaults to .hector/ but can be overridden (e.g. a cached
+    or shared checkout): an existing directory is reused as-is (no clone), a missing one
+    is cloned into. Only called when the run actually needs the source (see callers)."""
+    os.makedirs(ARTIFACTS_DIR, exist_ok=True)
+    renode_dir = renode_dir or os.path.join(ARTIFACTS_DIR, "renode")
+    integration_dir = integration_dir or os.path.join(ARTIFACTS_DIR, "renode-verilator-integration")
+
+    if not os.path.isdir(renode_dir):
+        print(f"[DEPS] Cloning Renode @ {git_tag} → {renode_dir} ...")
+        run_shell(f'git clone --depth 1 --branch "{git_tag}" --recurse-submodules '
+                  f'--shallow-submodules "{RENODE_REPO_URL}" "{renode_dir}"')
+    else:
+        print(f"[DEPS] Renode present at {renode_dir} (delete to refresh).")
+
+    if not os.path.isdir(integration_dir):
+        print(f"[DEPS] Cloning renode-verilator-integration → {integration_dir} ...")
+        run_shell(f'git clone --recurse-submodules "{INTEGRATION_REPO_URL}" "{integration_dir}"')
+    else:
+        print(f"[DEPS] Integration present at {integration_dir} (delete to refresh).")
+
+    return renode_dir, integration_dir
+
+
+def framework_vars(renode_dir, integration_dir):
+    """Interpolation variables exposing the checkouts as container paths, bind-mounting
+    any that live outside the workspace so containers reach them at the same path."""
+    return {
+        "RENODE_DIR": register_dependency_mount(renode_dir),
+        "INTEGRATION_DIR": register_dependency_mount(integration_dir),
+    }

hector/docker.py

@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Thin wrappers around `docker run` for the things we launch in the container:
+module builds, interactive Renode simulation, renode-test runs, and the
+interactive robot REPL. Centralising them here keeps the mount/flag
+boilerplate in one place.
+"""
+
+import os
+import subprocess
+
+from . import core
+from .core import to_container_path
+
+
+def _xdisplay_args():
+    xauth = os.environ.get("XAUTHORITY", f"{os.environ.get('HOME')}/.Xauthority")
+    display = os.environ.get("DISPLAY", ":0")
+    # Pin XAUTHORITY explicitly so X auth still resolves when the container runs as the
+    # host UID (see _user_flags) rather than the image's baked-in user with HOME set.
+    return ["-e", f"DISPLAY={display}", "-e", "XAUTHORITY=/home/developer/.Xauthority",
+            "-v", f"{xauth}:/home/developer/.Xauthority"]
+
+
+def _user_flags():
+    """Run the container as the host user, so files it writes into the bind-mounted
+    workspace are owned by them on the host — not root. Used by the shell/build and
+    renode-test containers.
+
+    Placed BEFORE _runtime_flags at the renode call sites: docker honours the last
+    `-u`, so a mapping that legitimately needs root (e.g. a tap device, via
+    runtime.root) still wins. The module-build container (run_build) is the one
+    exception — it needs root for apt-get and chowns its artifacts back itself."""
+    return ["-u", f"{os.getuid()}:{os.getgid()}"]
+
+
+def _workspace_mount():
+    return ["-v", f"{os.getcwd()}:{core.WORKSPACE_MOUNT}", "-w", core.WORKSPACE_MOUNT]
+
+
+def _extra_mounts():
+    """Bind-mount out-of-tree dependency dirs (e.g. a shared Renode source cache) 1:1, so
+    containers reach them at the same path they have on the host. Empty for the common case."""
+    args = []
+    for d in core.EXTRA_MOUNTS:
+        args += ["-v", f"{d}:{d}"]
+    return args
+
+
+def _runtime_flags(runtime):
+    """Map aggregated RuntimeOptions (capabilities/devices/root, declared by mappings,
+    hubs, modules) to `docker run` flags.
+
+    A requested --device is dropped if absent on the host so `docker run` never fails;
+    and if devices were requested but none exist, no flags are applied at all (e.g. a
+    tap on a host without /dev/net/tun runs as the normal user in dummy mode rather than
+    needlessly as root)."""
+    if not runtime:
+        return []
+    present = [d for d in sorted(runtime.devices) if os.path.exists(d)]
+    if runtime.devices and not present:
+        return []
+    flags = []
+    if runtime.root:
+        flags += ["-u", "0:0"]
+    for cap in sorted(runtime.capabilities):
+        flags += ["--cap-add", cap]
+    for dev in present:
+        flags += ["--device", dev]
+    return flags
+
+
+def run_build(script, image):
+    """Run a build script in the container as root (for apt-get + cmake)."""
+    if core.NO_DOCKER:
+        subprocess.run(["bash", "-c", script], check=True)
+        return
+    subprocess.run(
+        ["docker", "run", "--rm", "-u", "0:0",
+         *_workspace_mount(), *_extra_mounts(), image, "bash", "-c", script],
+        check=True,
+    )
+
+
+def resolve_shell_image(item, default_image, label="shell"):
+    """Decide which image a shell step/test runs in, and log the decision.
+
+    - explicit ``image:``            → use it
+    - none                           → ``default_image`` (the run's renode image), logged
+                                       so the user sees the fallback
+    - under ``--no-docker``          → the image is irrelevant (the step runs on the host);
+                                       warn if one was set explicitly, since it's ignored
+
+    Returns the image string. Under --no-docker run_in_container ignores it, so the value
+    only matters for display there."""
+    image = item.get("image")
+    name = item.get("name") or label
+    if core.NO_DOCKER:
+        if image:
+            print(f"[WARN] --no-docker: image '{image}' for '{name}' is ignored; "
+                  "running on the host.")
+        return image or default_image
+    if not image:
+        print(f"[INFO] '{name}': no image set; using the default image '{default_image}'.")
+    return image or default_image
+
+
+def run_in_container(script, image, live=False):
+    """Run a shell script with the project bind-mounted at the workspace path, returning
+    (output, passed); streams when live=True. `set -ex` makes commands echo and the run
+    fail on the first error.
+
+    Backs the `shell` test type and the `build:` steps. Normally a one-off container of
+    `image`; under --no-docker the script runs directly on the host (the image is moot)."""
+    if core.NO_DOCKER:
+        cmd = ["bash", "-c", f"set -ex\n{script}"]
+    else:
+        cmd = ["docker", "run", "--rm", *_workspace_mount(), *_extra_mounts(),
+               *_user_flags(), image, "sh", "-c", f"set -ex\n{script}"]
+    if live:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT, text=True)
+        lines = []
+        for line in proc.stdout:
+            print(line, end="", flush=True)
+            lines.append(line)
+        proc.wait()
+        return "".join(lines), proc.returncode == 0
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    if proc.stdout:
+        print(proc.stdout, end="")
+    return proc.stdout, proc.returncode == 0
+
+
+def build_renode_argv(resc_filename, image, ports=None, extra_args=None, runtime=None):
+    """Build the exact argv that would run an emulation resc — the local `renode`
+    invocation under --no-docker, otherwise the full `docker run ... renode` command.
+
+    Returning the command (instead of running it) is what lets the `export` command
+    print it verbatim and keeps run_renode a one-liner around this builder.
+    runtime: aggregated RuntimeOptions whose docker flags (caps/devices/root) are added."""
+    if core.NO_DOCKER:
+        return ["renode", resc_filename] + (extra_args or [])
+    cmd = ["docker", "run", "-it", "--rm", *_xdisplay_args(), "--net=host",
+           *_workspace_mount(), *_extra_mounts(), *_user_flags(), *_runtime_flags(runtime)]
+    for port in (ports or []):
+        cmd += ["-p", f"{port}:{port}"]
+    cmd += [image, "renode", resc_filename] + (extra_args or [])
+    return cmd
+
+
+def run_renode(resc_filename, image, ports=None, extra_args=None, output_dir="results", runtime=None):
+    """Run an emulation resc interactively. In debug mode, expose GDB ports.
+    runtime: aggregated RuntimeOptions whose docker flags (caps/devices/root) are added."""
+    if not core.NO_DOCKER:
+        os.makedirs(os.path.join(os.getcwd(), output_dir), exist_ok=True)
+    subprocess.run(build_renode_argv(resc_filename, image, ports, extra_args, runtime))
+
+
+def run_renode_test(robot_file_host, image, results_dir_host,
+                    resc_host_path=None, extra_args=None, interactive=False, runtime=None):
+    """Run a .robot file with renode-test in the container. Returns True on pass.
+
+    resc_host_path: passed as --variable RESC:<path> so .robot files can load
+    the generated emulation script via ${RESC}.
+    extra_args: optional list of additional flags forwarded verbatim to renode-test.
+    interactive: attach stdin/tty and add --net=host (useful with Pause Execution
+    or when TCP ports need to be reachable from the host during the test).
+    """
+    os.makedirs(results_dir_host, exist_ok=True)
+    if core.NO_DOCKER:
+        cmd = ["renode-test", robot_file_host, "-r", results_dir_host]
+        if resc_host_path:
+            cmd += ["--variable", f"RESC:{resc_host_path}"]
+        if extra_args:
+            cmd += extra_args
+        result = subprocess.run(cmd)
+        return result.returncode == 0
+    flags = ["-it"] if interactive else []
+    net = ["--net=host"] if interactive else []
+    display = _xdisplay_args() if interactive else []
+    cmd = ["docker", "run", "--rm", *flags, *net, *display,
+           *_workspace_mount(), *_extra_mounts(), *_user_flags(), *_runtime_flags(runtime),
+           image, "renode-test", to_container_path(robot_file_host),
+           "-r", to_container_path(results_dir_host)]
+    if resc_host_path:
+        cmd += ["--variable", f"RESC:{to_container_path(resc_host_path)}"]
+    if extra_args:
+        cmd += extra_args
+    result = subprocess.run(cmd)
+    return result.returncode == 0