qemu-compose 0.1.0__py3-none-any.whl

qemu_compose/jsonlisp.py

@@ -0,0 +1,254 @@
+
+# NOTE: the following code comes from https://github.com/Leden/jsonlisp with small modification for this project
+# but I have no idea why the original repo is 404 now, so I just leave a comment here to express thanks to Leden
+
+import json
+import logging
+import operator
+from collections import ChainMap
+import ast
+
+logger = logging.getLogger(__name__)
+
+Symbol = str
+List = list
+Number = (int, float)
+Dict = dict
+
+
+class Env(ChainMap):
+    def __json__(self):
+        return self.maps
+
+
+class Proc:
+    def __init__(self, params, body, env):
+        self.params = params
+        self.body = body
+        self.env = env
+
+    def __call__(self, *args):
+        env = Env(dict(zip(self.params, args)), self.env)
+        return interp(self.body, env)
+
+    def __json__(self):
+        return ["lambda", self.params, self.body]
+
+
+class Macro:
+    def __init__(self, params, body, env):
+        self.params = params
+        self.body = body
+        self.env = env
+
+    def expand(self, *args, env):
+        static_env = Env(dict(zip(self.params, args)), self.env)
+        exp = interp(self.body, static_env)
+        return interp(exp, env)
+
+    def __json__(self):
+        return ["macro", self.params, self.body]
+
+
+builtins = {
+    # Operators
+    "*": operator.mul,
+    "+": operator.add,
+    "-": operator.sub,
+    "/": operator.truediv,
+    "<": operator.lt,
+    "<=": operator.le,
+    "=": operator.eq,
+    ">": operator.gt,
+    ">=": operator.ge,
+    "^": operator.xor,
+    "and": operator.and_,
+    "contains": operator.contains,
+    "in": lambda x, y: x in y,
+    "is": operator.is_,
+    "is-not": operator.is_not,
+    "not": operator.not_,
+    "or": operator.or_,
+    "xor": lambda x, y: bool(x) ^ bool(y),  # logical XOR
+    # Typechecks
+    "dict?": lambda x: isinstance(x, Dict),
+    "list?": lambda x: isinstance(x, List),
+    "macro?": lambda x: isinstance(x, Macro),
+    "null?": lambda x: x is None,
+    "number?": lambda x: isinstance(x, Number),
+    "proc?": callable,
+    "symbol?": lambda x: isinstance(x, Symbol),
+    # List functions
+    "begin": lambda *x: x[-1],
+    "cons": lambda x, y: [x, *y],
+    "head": lambda x: x[0],
+    "len": len,
+    "list": lambda *x: list(x),
+    "map": lambda *args: list(map(*args)),
+    "range": lambda *x: list(range(*x)),
+    "tail": lambda x: x[1:],
+    # Dict functions
+    "dict": lambda x: dict(x),
+    "dict-del": lambda d, k: {_k: d[_k] for _k in d if k != _k},
+    "dict-get": lambda d, k: d.get(k),
+    "dict-items": lambda x: list(x.items()),
+    "dict-set": lambda d, k, v: {**d, k: v},
+    # Misc functions
+    "apply": lambda proc, args: proc(*args),
+    "print": print,
+    "literal": ast.literal_eval,
+    "str": str,
+    "format": lambda x, *args: x % tuple(args),
+}
+
+
+std_lib = [
+    "list",
+    [
+        "def",
+        "defmacro",
+        [
+            "macro",
+            ["name", "params", "body"],
+            [
+                "list",
+                ["quote", "def"],
+                "name",
+                ["list", ["quote", "macro"], "params", "body"],
+            ],
+        ],
+    ],
+    [
+        "defmacro",
+        "defproc",
+        ["name", "params", "body"],
+        [
+            "list",
+            ["quote", "def"],
+            "name",
+            ["list", ["quote", "lambda"], "params", "body"],
+        ],
+    ],
+]
+
+
+def default_env():
+    env = Env()
+    env.update(builtins)
+    env['key_up'] = "\x1b[A"
+    env['key_down'] = "\x1b[B"
+    env['key_right'] = "\x1b[C"
+    env['key_left'] = "\x1b[D"
+    env['key_home'] = "\x1b[H"
+    env['key_end'] = "\x1b[F"
+    env['key_ctrl_space'] = '\x00'
+    env['key_escape'] = '\x1b'
+    env['key_tab'] = '\t'
+    env['key_enter'] = '\n'
+    env['key_backspace'] = '\x7f'
+    interp(std_lib, env)
+    return env
+
+
+def parse(line):
+    return json.loads(line)
+
+
+def parse_file(path):
+    return json.load(path)
+
+
+def unparse(expr):
+    return json.dumps(expr, default=to_json)
+
+
+def to_json(obj):
+    if hasattr(obj, "__json__"):
+        return obj.__json__()
+    return repr(obj)
+
+
+def interp(x, env):
+    # a shortcut syntax for human readability
+    if isinstance(x, Dict) and len(x) == 1:
+        for k in x:
+            v = x[k]
+            if isinstance(v, List) or isinstance(v, Dict):
+                v = interp(v, env)
+
+            if not isinstance(v, List):
+                v = [v]
+
+            f = env[k]
+            logger.info('CALL_STEP: %s %s' % (f.__name__, v))
+            return f(*v)
+
+    if isinstance(x, Symbol):
+        if x.startswith('key_') and len(x) == 5:
+            return x[4]
+        return env[x]
+
+    if not isinstance(x, List):
+        return x
+
+    if x == []:
+        return x
+
+    if x[0] == "quote" or x[0] == "'":
+        _, exp = x
+        return exp
+
+    if x[0] == "flat_quote" or x[0] == "_'":
+        _, *exp = x
+        return exp
+
+    if x[0] == "if":
+        _, test, conseq, alt = x
+        exp = conseq if interp(test, env) else alt
+        return interp(exp, env)
+
+    if x[0] == "def":
+        _, var, exp = x
+        val = env[var] = interp(exp, env)
+        return val
+
+    if x[0] == "lambda":
+        _, params, body = x
+        return Proc(params, body, env)
+
+    if x[0] == "macro":
+        _, params, body = x
+        return Macro(params, body, env)
+
+    proc_or_macro_exp, *args = x
+    proc = interp(proc_or_macro_exp, env)
+    if isinstance(proc, Macro):
+        return proc.expand(*args, env=env)
+    else:
+        args = [interp(exp, env) for exp in args]
+        if callable(proc):
+            if proc_or_macro_exp != "begin":
+                logger.info('CALL_STEP: %s(%s) %s' % (proc.__name__, proc_or_macro_exp, args))
+        else:
+            logger.error('CALL_STEP:ERROR: function not callable: %s(%s) %s' % (proc, proc_or_macro_exp, args))
+        return proc(*args)
+
+def repl(prompt=r"{λ}> "):
+    env = default_env()
+
+    while True:
+        try:
+            line = input(prompt)
+            print(unparse(interp(parse(line), env)))
+        except EOFError:
+            print("Bye")
+            break
+        except Exception as err:
+            print(repr(err))
+
+
+def run_file(path):
+    env = default_env()
+
+    with open(path) as f:
+        print(unparse(interp(parse_file(f), env)))

qemu_compose/log_tool.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+import ast
+import sys
+import socket
+from zio import zio
+
+def parse(log_path:str):
+    buf = [bytearray(), bytearray()]
+    ret = []
+
+    for line in open(log_path):
+        if 'IO.recv(' in line:
+            if buf[1]:
+                ret.append((1, bytes(buf[1])))
+                buf[1] = bytearray()
+            i = line.index(' -> ')
+            content = line[i+4:-1].strip()
+            assert (content.startswith('b"') and content.endswith('"')) or (content.startswith("b'") and content.endswith("'"))
+            buf[0].extend(ast.literal_eval(content))
+        elif 'IO.send(' in line:
+            if buf[0]:
+                ret.append((0, bytes(buf[0])))
+                buf[0] = bytearray()
+            i = line.index(' -> ')
+            content = line[i+4:-1].strip()
+            assert (content.startswith('b"') and content.endswith('"')) or (content.startswith("b'") and content.endswith("'"))
+            buf[1].extend(ast.literal_eval(content))
+        else:
+            continue
+
+    if buf[0]:
+        ret.append((0, bytes(buf[0])))
+    if buf[1]:
+        ret.append((1, bytes(buf[1])))
+    return ret
+
+def serve(logs, addr=None):
+    server = socket.socket()
+    if addr is None:
+        addr = ('0.0.0.0', 1111)
+    server.bind(addr)
+    server.listen(1)
+    s, _ = server.accept()
+    io = zio(s, print_read=False, print_write=False, timeout=3600)
+    for d, c in logs:
+        if d == 1:
+            print('EXPECT', c)
+            io.read_until(c)
+        else:
+            print('SEND', c)
+            io.write(c)
+    io.close()
+
+def main(name, log_path):
+    logs = parse(log_path)
+    if name == 'parse':
+        for direction, content in logs:
+            print('<<<<----' if direction else '---->>>>', content)
+    elif name == 'serve':
+        serve(logs)
+
+if __name__ == '__main__':
+    main(sys.argv[1], sys.argv[2])

qemu_compose/main.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+from typing import List, Optional
+import os
+import re
+import sys
+import yaml
+import logging
+import shutil
+import threading
+import tty
+import subprocess
+from functools import partial
+from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
+
+from .qemu.machine import QEMUMachine
+from .qemu.machine.machine import AbnormalShutdown
+from .jsonlisp import default_env, interp
+
+from .zio import zio, write_debug, select_ignoring_useless_signal, ttyraw
+
+logger = logging.getLogger("resident")
+
+class HttpServer:
+    def __init__(self, listen:str, port:int, root:str):
+        self.listen = listen
+        self.port = port
+        self.root = root
+
+    def start(self):
+        http_handler = partial(SimpleHTTPRequestHandler, directory=self.root)
+        server = ThreadingHTTPServer((self.listen, self.port), http_handler)
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+
+class StreamWrapper:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __getattr__(self, name):
+        return getattr(self.obj, name)
+
+    def write(self, s):
+        if isinstance(s, str):
+            s = s.encode()
+        self.obj.write(s)
+
+class Terminal(object):
+    def __init__(self, fd, log_path=None):
+        self.fd = fd
+
+        if isinstance(log_path, str):
+            self.debug_file = open(log_path, "wb") if log_path else None
+        else:
+            self.debug_file = log_path
+
+        self.io = zio(fd, print_write=False, logfile=sys.stdout, debug=self.debug_file, timeout=3600)
+
+        self.term_feed_running = False
+        self.term_feed_drain_thread = None
+
+        if not os.isatty(0):
+            raise Exception('resident.Terminal must run in a UNIX 98 style pty/tty')
+
+    def term_feed_loop(self):
+        logger.info('Terminal.term_feed_loop started...')
+        while self.term_feed_running:
+            r, _, _ = select_ignoring_useless_signal([0], [], [], 0.2)
+
+            if 0 in r:
+                data = os.read(0, 1024)
+                if data:
+                    logger.info('Terminal.term_feed_loop received(%d) -> %s' % (len(data), data))
+                    self.io.write(data)
+
+        logger.info('Terminal.term_feed_loop finished.')
+
+    def run_batch(self, cmds:List, env_variables=None):
+        if not isinstance(cmds, list):
+            raise ValueError("cmds must be a list")
+        
+        current_tty_mode = tty.tcgetattr(0)[:]
+        ttyraw(0)
+
+        try:
+            self.term_feed_running = True
+            self.term_feed_drain_thread = threading.Thread(target=self.term_feed_loop)
+            self.term_feed_drain_thread.daemon = True
+            self.term_feed_drain_thread.start()
+            
+            io = self.io
+
+            if self.debug_file:
+                write_debug(self.debug_file, b'run_batch: cmds = %r' % cmds)
+
+            transpiled_cmds = ['begin'] + cmds
+
+            env = default_env()
+
+            env['read_until'] = io.read_until
+            env['write'] = io.write
+            env['writeline'] = io.writeline
+            env['wait'] = io.read_until_timeout
+            env['RegExp'] = lambda x: re.compile(x.encode())
+            env['interact'] = self.interact
+
+            if env_variables:
+                env.update(env_variables)
+
+            interp(transpiled_cmds, env)
+
+        finally:
+            tty.tcsetattr(0, tty.TCSAFLUSH, current_tty_mode)
+
+    def interact(self, buffered:Optional[bytes]=None, raw_mode=False):
+
+        self.term_feed_running = False
+        if self.term_feed_drain_thread is not None:
+            self.term_feed_drain_thread.join()
+
+        self.io.interactive(raw_mode=raw_mode, buffered=buffered)
+
+def extract_format_or_default(mapping:dict, key:str, env:dict, default=None):
+    value = mapping.get(key) if mapping else key
+    if value:
+        # FIXME: format has security issues
+        return str(value).format(**env)
+    return default
+
+def run(config_path, log_path=None, env_update=None):
+
+    if log_path:
+        debug_file = open(log_path, "wb")
+        logging.basicConfig(level=logging.DEBUG, stream=StreamWrapper(debug_file))
+    else:
+        debug_file = None
+
+    config: dict
+    with open(config_path) as f:
+        config = yaml.safe_load(f)
+
+    name = config.get('name')
+
+    binary = config.get('binary', shutil.which('qemu-system-x86_64'))
+    if not binary:
+        raise Exception('qemu binary not found')
+    
+    cwd = os.path.normpath(os.path.dirname(config_path))
+    term_size = os.get_terminal_size()
+
+    env = {
+        'CWD': cwd,
+        'GATEWAY_IP': '10.0.2.2',   # qemu user network default gateway ip
+        'TERM_ROWS': term_size.lines,
+        'TERM_COLS': term_size.columns,
+    }
+
+    if config.get('env'):
+        for k in config.get('env'):
+            env[k] = config.get('env')[k]
+
+    os.chdir(env['CWD'])
+    
+    http_port = None
+    if config.get('http_serve'):
+        http_serve_config:dict = config.get('http_serve')
+        http_listen = extract_format_or_default(http_serve_config, 'listen', env, default='0.0.0.0')
+        http_port = int(extract_format_or_default(http_serve_config, 'port', env, default=8888))
+        http_root = extract_format_or_default(http_serve_config, 'root', env, default=env['CWD'])
+
+        http_server = HttpServer(http_listen, http_port, http_root)
+        http_server.start()
+        logger.info('HTTP server started on %s:%d, serving %s' % (http_listen, http_port, http_root))
+
+    if http_port is not None:
+        env['HTTP_PORT'] = http_port
+        access_ip = extract_format_or_default(config.get('http_serve'), 'access_ip', env, default=env['GATEWAY_IP']) if config.get('http_serve') else env['GATEWAY_IP']
+        env['HTTP_HOST'] = access_ip
+
+    if env_update:
+        env.update(env_update)
+
+    if config.get('before_script'):
+        for line in config.get('before_script'):
+            command = extract_format_or_default(None, line, env)
+            if command:
+                subprocess.run(command.strip(), shell=True, check=True)
+
+    default_args = {
+        'cpu': 'max',
+        'machine': 'type=q35',
+        'accel': 'kvm',
+        'm': '1G',
+        'smp': '1',
+    }
+
+    for block in config.get('args'):
+        for key in block:
+            val = extract_format_or_default(block, key, env)
+            if key in default_args:
+                default_args[key] = val
+
+    args = []
+    for key in default_args:
+        args.append('-' + key)
+        args.append(default_args[key])
+
+    for block in config.get('args'):
+        for key in block:
+            val = extract_format_or_default(block, key, env)
+            if key in default_args:
+                continue
+            args.append('-' + key)
+            if val is not None:
+                args.append(val)
+
+    vm = QEMUMachine(binary, args=args, name=name)
+    vm.add_monitor_null()
+    vm.set_qmp_monitor(True)
+    vm.set_console(console_chardev='socket', device_type='isa-serial')
+
+    try:
+        vm.launch()
+
+        term = Terminal(vm.console_file, debug_file)
+
+        boot_commands = config.get('boot_commands')
+        if boot_commands:
+            term.run_batch(boot_commands, env_variables=env)
+        else:
+            term.interact(raw_mode=True)
+
+        if config.get('after_script'):
+            for line in config.get('after_script'):
+                command = extract_format_or_default(None, line, env)
+                if command:
+                    subprocess.run(command.strip(), shell=True, check=True)
+
+    except KeyboardInterrupt:
+        logger.warning("Keyboard interrupt, shutting down vm...")
+    finally:
+        try:
+            if vm.is_running():
+                vm.shutdown(hard=True)
+        except AbnormalShutdown:
+            logger.error('abnormal shutdown exception')
+        finally:
+            vm._load_io_log()
+            logger.info('vm.process_io_log = %r' % (vm.get_log(), ))
+
+def guess_conf_path(p:str | None):
+    if p:
+        return p
+    for f in ["qemu-compose.yml", "qemu-compose.yaml"]:
+        if os.path.exists(f):
+            return f
+    return None
+
+def cli():
+    import argparse
+    parser = argparse.ArgumentParser(description="Define and run QEMU VM with qemu")
+    parser.add_argument('command', type=str, help='command to run')
+    parser.add_argument('-f', "--file", type=str, help='Compose configuration files')
+    args = parser.parse_args()
+    if args.command == "up":
+        conf_path = guess_conf_path(args.file)
+        if not conf_path:
+            print("qemu-compose.yml not found", file=sys.stderr)
+            sys.exit(1)
+        run(conf_path)

qemu_compose/qemu/__init__.py

@@ -0,0 +1,3 @@
+
+# code under this package comes from https://gitlab.com/qemu-project/qemu/-/tree/master/python/qemu with small modification
+# they are licensed under https://gitlab.com/qemu-project/qemu/-/blob/master/LICENSE

qemu_compose/qemu/machine/__init__.py

@@ -0,0 +1,36 @@
+"""
+QEMU development and testing library.
+
+This library provides a few high-level classes for driving QEMU from a
+test suite, not intended for production use.
+
+ | QEMUQtestProtocol: send/receive qtest messages.
+ | QEMUMachine: Configure and Boot a QEMU VM
+ | +-- QEMUQtestMachine: VM class, with a qtest socket.
+
+"""
+
+# Copyright (C) 2020-2021 John Snow for Red Hat Inc.
+# Copyright (C) 2015-2016 Red Hat Inc.
+# Copyright (C) 2012 IBM Corp.
+#
+# Authors:
+#  John Snow <jsnow@redhat.com>
+#  Fam Zheng <fam@euphon.net>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+
+# pylint: disable=import-error
+# see: https://github.com/PyCQA/pylint/issues/3624
+# see: https://github.com/PyCQA/pylint/issues/3651
+from .machine import QEMUMachine
+from .qtest import QEMUQtestMachine, QEMUQtestProtocol
+
+
+__all__ = (
+    'QEMUMachine',
+    'QEMUQtestProtocol',
+    'QEMUQtestMachine',
+)