sydost-tracekit 0.0.1__py3-none-any.whl

sydost_tracekit/__init__.py

@@ -0,0 +1,96 @@
+"""Reference memory traces of virtual machines and accuracy of accumulated-deficit estimates.
+
+Modules
+-------
+recorder
+    Guest recorder, Python standard library only; copied into a guest as one file.
+collector
+    Host collector of libvirt memory statistics (optional extra ``kvm``).
+trace
+    Trace of demand on a uniform grid, CSV format with validation.
+load
+    Recordings of the guest recorder turned into traces.
+clock
+    Host and guest clock alignment (Cristian's algorithm, drift).
+reconstruct
+    Hold and linear reconstruction between samples.
+integral
+    Accumulated deficit, its estimate from samples, exact positive-part integral.
+bounds
+    A-priori bounds (hold, linear), the refined hold bound, the sampling plan.
+replay, tune
+    Replay of a controller on a trace, Pareto front and selection under constraints.
+synthetic
+    Synthetic scenarios A, B, C for debugging.
+cli
+    Command line ``sydost-tracekit``.
+"""
+
+from .bounds import (
+    Constants,
+    Plan,
+    estimate_constants,
+    hold_error_bound,
+    hold_error_bounds_refined,
+    linear_error_bound,
+    plan_sampling,
+    required_step,
+)
+from .clock import ClockOffset, ClockSample, estimate_offset, fit_drift, guest_to_host
+from .integral import (
+    deficit,
+    deficit_integral,
+    estimate_from_samples,
+    positive_part_integral,
+    sampling_study,
+    total_variation,
+)
+from .load import demand, read_recorder_csv, recordings_to_trace
+from .reconstruct import hold, linear
+from .replay import ReserveController, RunResult, simulate
+from .synthetic import SCENARIOS, VmProfile, synthetic_trace
+from .trace import Trace, TraceFormatError, load_trace, save_trace
+from .tune import pareto_front, select, sweep
+
+__version__ = "0.0.1"
+
+__all__ = [
+    "SCENARIOS",
+    "ClockOffset",
+    "ClockSample",
+    "Constants",
+    "Plan",
+    "ReserveController",
+    "RunResult",
+    "Trace",
+    "TraceFormatError",
+    "VmProfile",
+    "__version__",
+    "deficit",
+    "deficit_integral",
+    "demand",
+    "estimate_constants",
+    "estimate_from_samples",
+    "estimate_offset",
+    "fit_drift",
+    "guest_to_host",
+    "hold",
+    "hold_error_bound",
+    "hold_error_bounds_refined",
+    "linear",
+    "linear_error_bound",
+    "load_trace",
+    "pareto_front",
+    "plan_sampling",
+    "positive_part_integral",
+    "read_recorder_csv",
+    "recordings_to_trace",
+    "required_step",
+    "sampling_study",
+    "save_trace",
+    "select",
+    "simulate",
+    "sweep",
+    "synthetic_trace",
+    "total_variation",
+]

sydost_tracekit/bounds.py

@@ -0,0 +1,202 @@
+"""A-priori and refined error bounds of the accumulated-deficit estimate.
+
+Statement 4.1 (hold). |D_l - D| <= l * V(d), where V is the total variation of
+the demand. It follows from |x^+ - y^+| <= |x - y|. It is valid, but loose: the
+bound counts variation at all times, also when the demand stays below the
+allocation.
+
+Refined hold bound (this package). The error of a sample at time t_j on the
+step [t_j, t_j + l) is nonzero only where the demand or the held value exceeds
+the allocation. Therefore, on the fine grid of the reference trace,
+
+    -sum dt (d_k - x_j)^+ 1{d_k > a_k}  <=  D_l - D  <=  sum dt (x_j - d_k)^+ 1{x_j > a_k},
+
+where x_j = d(t_j) is the held value of the fine point k. The sharper form used
+in the code is the pointwise one: the error of a point is at least
+-(d_k - max(x_j, a_k))^+ and at most (x_j - max(d_k, a_k))^+. The lower bound
+counts only the rises of the demand above the allocation, the upper bound only
+the falls of the held value above it. Both are computed from the reference trace
+and the allocation, which are known.
+
+Statement 4.2 (linear, second order). If d is L-Lipschitz with K kinks and
+|d''| <= M away from the kinks, then |D_lin - D| <= l^2 (T M / 12 + K L / 3).
+The constants are estimated from a reference trace by :func:`estimate_constants`;
+for noisy traces M and K are large and the bound is loose. The bound is a
+proof-sketch statement; see docs/en/bounds.md for the derivation.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+import numpy as np
+
+from .integral import total_variation
+
+
+def hold_error_bound(d_fine: np.ndarray, ell: float) -> float:
+    """Statement 4.1: l * V(d), with V computed on the fine grid.
+
+    Examples
+    --------
+    >>> hold_error_bound([[0.0, 2.0, 1.0]], 0.5)
+    1.5
+    """
+    return ell * total_variation(np.asarray(d_fine, float))
+
+
+def hold_error_bounds_refined(
+    d: np.ndarray, a: np.ndarray, dt: float, ell: float
+) -> tuple[float, float]:
+    """Refined lower and upper bounds of D_l - D for the hold estimate.
+
+    ``d`` and ``a`` are on the fine grid of step ``dt``; ``ell`` is a multiple of
+    ``dt`` (rounded). Returns ``(lower, upper)`` with lower <= D_l - D <= upper.
+
+    Examples
+    --------
+    >>> d = np.array([[0.0, 4.0, 0.0, 0.0]])
+    >>> a = np.full((1, 4), 3.0)
+    >>> hold_error_bounds_refined(d, a, 1.0, 2.0)
+    (-1.0, 0.0)
+    """
+    d = np.atleast_2d(np.asarray(d, float))
+    a = np.atleast_2d(np.asarray(a, float))
+    if d.shape != a.shape:
+        raise ValueError("d and a must have the same shape")
+    m = max(1, round(ell / dt))
+    idx = (np.arange(d.shape[1]) // m) * m
+    x = d[:, idx]
+    rise = np.maximum(d - np.maximum(x, a), 0.0)
+    fall = np.maximum(x - np.maximum(d, a), 0.0)
+    return -dt * float(rise.sum()), dt * float(fall.sum())
+
+
+def linear_error_bound(ell: float, T: float, M: float, K: int, L: float) -> float:
+    """Statement 4.2: l^2 (T M / 12 + K L / 3) for one machine.
+
+    Examples
+    --------
+    >>> linear_error_bound(0.5, 10.0, 1.2, 0, 3.0)
+    0.25
+    """
+    return ell * ell * (T * M / 12.0 + K * L / 3.0)
+
+
+@dataclass(frozen=True)
+class Constants:
+    """Constants of the bounds of one trace, summed over the machines.
+
+    V is the total variation, L and M are the largest over machines, K is the
+    total number of kinks; ``linear_C`` is sum over machines of T M / 12 + K L / 3.
+    """
+
+    V: float
+    L: float
+    M: float
+    K: int
+    linear_C: float
+
+
+def estimate_constants(d: np.ndarray, dt: float, kink_tol: float | None = None) -> Constants:
+    """Estimate V, L, M, K of the bounds from a fine reference trace.
+
+    A kink is a change of slope larger than ``kink_tol`` (MiB/s); by default it is
+    10 % of the largest slope of the machine. M is the largest second difference
+    divided by dt^2, a conservative estimate.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> c = estimate_constants(np.array([[0.0, 1.0, 2.0, 3.0]]), 1.0)
+    >>> c.V, c.L, c.M, c.K
+    (3.0, 1.0, 0.0, 0)
+    """
+    d = np.atleast_2d(np.asarray(d, float))
+    slope = np.diff(d, axis=1) / dt
+    if slope.shape[1] < 2:
+        raise ValueError("the trace needs at least three points")
+    second = np.diff(slope, axis=1) / dt
+    V = total_variation(d)
+    L = float(np.max(np.abs(slope)))
+    M = float(np.max(np.abs(second)))
+    K = 0
+    linear_C = 0.0
+    T = d.shape[1] * dt
+    for i in range(d.shape[0]):
+        Li = float(np.max(np.abs(slope[i])))
+        tol = kink_tol if kink_tol is not None else 0.1 * Li
+        ki = int(np.sum(np.abs(np.diff(slope[i])) > tol))
+        Mi = float(np.max(np.abs(second[i])))
+        K += ki
+        linear_C += T * Mi / 12.0 + ki * Li / 3.0
+    return Constants(V=V, L=L, M=M, K=K, linear_C=linear_C)
+
+
+def required_step(target: float, method: str, c: Constants) -> float:
+    """Largest step l for which the bound of ``method`` does not exceed ``target``.
+
+    Parameters
+    ----------
+    target : float
+        Admissible absolute error of D, MiB*s (positive).
+    method : str
+        ``"hold"`` (l V <= target) or ``"linear"`` (l^2 C <= target).
+    c : Constants
+        Constants of the trace.
+
+    Examples
+    --------
+    >>> c = Constants(V=100.0, L=1.0, M=0.0, K=0, linear_C=0.0)
+    >>> required_step(50.0, "hold", c)
+    0.5
+    """
+    if target <= 0:
+        raise ValueError("the target error must be positive")
+    if method == "hold":
+        return math.inf if c.V == 0 else target / c.V
+    if method == "linear":
+        return math.inf if c.linear_C == 0 else math.sqrt(target / c.linear_C)
+    raise ValueError(f"unknown method {method!r}")
+
+
+@dataclass(frozen=True)
+class Plan:
+    """Sampling plan for a target error: method, step, rate, and the bound at that step."""
+
+    method: str
+    step_s: float
+    rate_hz: float
+    bound_MiBs: float
+
+
+def plan_sampling(target: float, c: Constants) -> Plan:
+    """Choose the method that needs the fewest samples for the target error.
+
+    The method with the largest admissible step gives the lowest rate. The step
+    is the largest one for which the bound does not exceed the target. The rate
+    is 1 / step, per machine.
+
+    Examples
+    --------
+    >>> c = Constants(V=100.0, L=1.0, M=0.0, K=0, linear_C=1e9)
+    >>> p = plan_sampling(50.0, c)
+    >>> (p.method, p.step_s, p.rate_hz)
+    ('hold', 0.5, 2.0)
+    """
+    best: Plan | None = None
+    for method in ("hold", "linear"):
+        step = required_step(target, method, c)
+        if math.isinf(step):
+            bound = 0.0
+        elif method == "hold":
+            bound = step * c.V
+        else:
+            bound = step * step * c.linear_C
+        plan = Plan(method, step, 0.0 if math.isinf(step) else 1.0 / step, bound)
+        if best is None or plan.step_s > best.step_s:
+            best = plan
+    if best is None:  # pragma: no cover - the loop always sets it
+        raise RuntimeError("no sampling plan")
+    return best

sydost_tracekit/cli.py

@@ -0,0 +1,131 @@
+"""Command line: ``sydost-tracekit record | integrate | bound | tune``.
+
+record
+    Run the guest recorder on this machine (the same code as recorder.py).
+integrate
+    Estimate D from samples of a reference trace for several steps and methods.
+bound
+    Constants of a reference trace and a-priori bounds for a step or a target error.
+tune
+    Replay polling intervals on a trace, print the Pareto front and the selection.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+
+from . import __version__
+from .bounds import estimate_constants, hold_error_bound, plan_sampling
+from .integral import sampling_study
+from .recorder import record
+from .trace import load_trace
+from .tune import pareto_front, select, sweep
+
+
+def _floats(text: str) -> list[float]:
+    return [float(x) for x in text.split(",") if x.strip()]
+
+
+def _cmd_record(args: argparse.Namespace) -> int:
+    summary = record(args.out, args.hz, args.duration, args.root)
+    with open(args.out + ".json", "w", encoding="utf-8") as fh:
+        json.dump(summary, fh, ensure_ascii=False, indent=1)
+    print(json.dumps(summary, ensure_ascii=False))
+    return 0
+
+
+def _cmd_integrate(args: argparse.Namespace) -> int:
+    ref = load_trace(args.trace)
+    alloc = load_trace(args.alloc)
+    if alloc.d.shape != ref.d.shape or alloc.dt != ref.dt:
+        print("error: the allocation must have the grid and machines of the trace", file=sys.stderr)
+        return 2
+    methods = tuple(m.strip() for m in args.methods.split(","))
+    rows = sampling_study(ref, alloc.d, _floats(args.steps), methods)
+    print(f"D_true = {rows[0]['D_true_MiBs'] / 1024.0:.4f} GiB*s")
+    header = ["step_s", *[f"{m}_rel_pct" for m in methods]]
+    print("\t".join(header))
+    for row in rows:
+        print("\t".join(f"{row[k]:.4f}" for k in header))
+    return 0
+
+
+def _cmd_bound(args: argparse.Namespace) -> int:
+    ref = load_trace(args.trace)
+    c = estimate_constants(ref.d, ref.dt)
+    print(f"V = {c.V:.3f} MiB, L = {c.L:.3f} MiB/s, M = {c.M:.3f} MiB/s^2, K = {c.K}")
+    for ell in _floats(args.steps):
+        print(
+            f"step {ell:g} s: hold bound l*V = {hold_error_bound(ref.d, ell):.3f} MiB*s, "
+            f"linear bound = {ell * ell * c.linear_C:.3f} MiB*s"
+        )
+    if args.target is not None:
+        plan = plan_sampling(args.target, c)
+        print(
+            f"target {args.target:g} MiB*s: method {plan.method}, step {plan.step_s:.4g} s, "
+            f"rate {plan.rate_hz:.4g} Hz per machine"
+        )
+    return 0
+
+
+def _cmd_tune(args: argparse.Namespace) -> int:
+    ref = load_trace(args.trace)
+    rows = sweep(ref, _floats(args.intervals), theta=args.theta)
+    keys = ["cost", "D_GiBs"]
+    front = pareto_front(rows, keys)
+    print("interval_s\tcost\tP_pct\tD_GiBs\tpareto")
+    for r in rows:
+        mark = "*" if r in front else ""
+        print(f"{r['interval_s']:g}\t{r['cost']:.2f}\t{r['P_pct']:.3f}\t{r['D_GiBs']:.4f}\t{mark}")
+    chosen = select(rows, {"P_pct": args.alpha, "D_GiBs": args.dmax}, "cost")
+    if chosen is None:
+        print("no configuration satisfies the constraints")
+    else:
+        print(f"selected: interval {chosen['interval_s']:g} s, cost {chosen['cost']:.2f}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Return the argument parser of the command line."""
+    ap = argparse.ArgumentParser(prog="sydost-tracekit", description=__doc__.split("\n")[0])
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("record", help="run the guest recorder")
+    p.add_argument("--hz", type=float, default=10.0)
+    p.add_argument("--duration", type=float, default=600.0)
+    p.add_argument("--out", default="trace.csv")
+    p.add_argument("--root", default="/proc")
+    p.set_defaults(func=_cmd_record)
+
+    p = sub.add_parser("integrate", help="estimate D from samples")
+    p.add_argument("trace", help="reference trace (fine step)")
+    p.add_argument("--alloc", required=True, help="allocation on the same grid (trace format)")
+    p.add_argument("--steps", default="0.5,1,2", help="sample steps, s")
+    p.add_argument("--methods", default="hold,linear")
+    p.set_defaults(func=_cmd_integrate)
+
+    p = sub.add_parser("bound", help="a-priori bounds and the sampling rate for a target")
+    p.add_argument("trace", help="reference trace")
+    p.add_argument("--steps", default="1", help="steps for which the bounds are printed, s")
+    p.add_argument("--target", type=float, help="admissible error of D, MiB*s")
+    p.set_defaults(func=_cmd_bound)
+
+    p = sub.add_parser("tune", help="replay polling intervals and select one")
+    p.add_argument("trace", help="training trace")
+    p.add_argument("--intervals", default="1,2,4,8")
+    p.add_argument("--theta", type=float, default=64.0, help="deficit threshold, MiB")
+    p.add_argument("--alpha", type=float, default=0.5, help="admissible P(theta), %%")
+    p.add_argument("--dmax", type=float, default=float("inf"), help="admissible D, GiB*s")
+    p.set_defaults(func=_cmd_tune)
+    return ap
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the command line; return the exit status."""
+    args = build_parser().parse_args(argv)
+    result: int = args.func(args)
+    return result

sydost_tracekit/clock.py

@@ -0,0 +1,74 @@
+"""Alignment of the host and guest clocks.
+
+The guest recorder writes wall-clock time t_real_s of the guest. The host
+collector writes its own t_real_s. To compare them, the offset between the two
+clocks is estimated from request-response exchanges (Cristian's algorithm): the
+host notes the time before and after asking the guest for its clock.
+
+The error of the estimate is at most half of the round trip of the exchange
+used. The helper keeps the exchange with the smallest round trip.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class ClockSample:
+    """One exchange: host time before the request, guest time, host time after."""
+
+    t_send_host: float
+    t_guest: float
+    t_recv_host: float
+
+
+@dataclass(frozen=True)
+class ClockOffset:
+    """Estimated offset guest minus host, with its bound on the error, s."""
+
+    offset_s: float
+    uncertainty_s: float
+    n_samples: int
+
+
+def estimate_offset(samples: Sequence[ClockSample]) -> ClockOffset:
+    """Offset guest - host from the exchange with the smallest round trip.
+
+    Examples
+    --------
+    >>> s = [ClockSample(100.0, 105.2, 100.4), ClockSample(200.0, 205.0, 200.2)]
+    >>> est = estimate_offset(s)
+    >>> round(est.offset_s, 3), round(est.uncertainty_s, 3)
+    (5.0, 0.1)
+    """
+    if not samples:
+        raise ValueError("at least one exchange is needed")
+    best = min(samples, key=lambda s: s.t_recv_host - s.t_send_host)
+    rtt = best.t_recv_host - best.t_send_host
+    if rtt < 0:
+        raise ValueError("the host received the answer before sending the request")
+    midpoint = (best.t_send_host + best.t_recv_host) / 2.0
+    return ClockOffset(best.t_guest - midpoint, rtt / 2.0, len(samples))
+
+
+def guest_to_host(t_guest: np.ndarray, offset_s: float) -> np.ndarray:
+    """Convert guest wall-clock times to host time."""
+    return np.asarray(t_guest, float) - offset_s
+
+
+def fit_drift(host_t: np.ndarray, offsets: np.ndarray) -> tuple[float, float]:
+    """Linear fit offset = intercept + slope * host_time; slope in ppm.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> slope, intercept = fit_drift(np.array([0.0, 100.0]), np.array([1.0, 1.0001]))
+    >>> round(slope, 3)
+    1.0
+    """
+    slope, intercept = np.polyfit(np.asarray(host_t, float), np.asarray(offsets, float), 1)
+    return float(slope) * 1e6, float(intercept)

sydost_tracekit/collector.py

@@ -0,0 +1,131 @@
+"""Host collector: memory statistics of libvirt domains at a fixed frequency.
+
+The host path is slower than the guest recorder: QEMU asks the guest for its
+statistics once per period (whole seconds, at least 1 s), and memoryStats()
+returns the last snapshot (SPEC section 6.5). The collector writes the snapshot
+age ``age_s = t_real_s - last_update`` so that stale snapshots are visible.
+
+libvirt is optional and imported lazily: install with ``pip install
+sydost-tracekit[kvm]``. The tests use fake domains and need no libvirt.
+
+Example on the host::
+
+    from sydost_tracekit.collector import connect, collect
+
+    doms = connect("qemu:///system", ["vm1", "vm2"])
+    collect(doms, hz=1, duration=600, out="host_stats.csv")
+"""
+
+from __future__ import annotations
+
+import csv
+import time
+from collections.abc import Callable, Mapping
+from typing import Any, Protocol
+
+KEYS = [
+    "actual",
+    "available",
+    "usable",
+    "unused",
+    "disk_caches",
+    "last_update",
+    "rss",
+    "swap_in",
+    "swap_out",
+    "major_fault",
+    "minor_fault",
+]
+VIR_DOMAIN_AFFECT_LIVE = 1  # libvirt-domain.h: 1 << 0
+
+
+class Domain(Protocol):
+    """The part of the libvirt domain interface that the collector uses."""
+
+    def setMemoryStatsPeriod(self, period: int, flags: int) -> Any:  # noqa: N802
+        """Set the statistics period, s."""
+        ...
+
+    def memoryStats(self) -> Mapping[str, int]:  # noqa: N802
+        """Return the last memory statistics snapshot (KiB)."""
+        ...
+
+
+def connect(uri: str, names: list[str]) -> dict[str, Any]:
+    """Open a libvirt connection and look up the named domains.
+
+    Raises
+    ------
+    ImportError
+        If libvirt-python is not installed.
+    """
+    try:
+        import libvirt
+    except ImportError as err:  # pragma: no cover - needs libvirt on the host
+        raise ImportError("install the extra: pip install sydost-tracekit[kvm]") from err
+    conn = libvirt.open(uri)  # pragma: no cover - needs libvirt on the host
+    return {n: conn.lookupByName(n) for n in names}  # pragma: no cover
+
+
+def collect(
+    domains: Mapping[str, Domain],
+    hz: float,
+    duration: float,
+    out: str,
+    stats_period: int = 1,
+    set_period: bool = True,
+    clock: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+) -> int:
+    """Poll memoryStats() of all domains at frequency hz; return the number of rows.
+
+    Examples
+    --------
+    >>> class Fake:
+    ...     def setMemoryStatsPeriod(self, period, flags):
+    ...         pass
+    ...
+    ...     def memoryStats(self):
+    ...         return {"actual": 1024, "last_update": 0}
+    >>> import tempfile, os
+    >>> p = os.path.join(tempfile.mkdtemp(), "h.csv")
+    >>> collect({"vm1": Fake()}, hz=2.0, duration=1.0, out=p, set_period=False)
+    2
+    """
+    if hz <= 0 or duration <= 0:
+        raise ValueError("hz and duration must be positive")
+    if set_period:
+        for dom in domains.values():
+            dom.setMemoryStatsPeriod(int(stats_period), VIR_DOMAIN_AFFECT_LIVE)
+    period = 1.0 / hz
+    start = clock()
+    n = 0
+    k = 0
+    with open(out, "w", newline="", encoding="utf-8") as fh:
+        writer = csv.writer(fh)
+        writer.writerow(["t_mono_s", "t_real_s", "vm", *KEYS, "age_s"])
+        while True:
+            now = clock() - start
+            if now >= duration:
+                break
+            due = k * period
+            if now < due:
+                sleep(due - now)
+                continue
+            for name, dom in domains.items():
+                st = dom.memoryStats()
+                real = time.time()
+                last = st.get("last_update", "")
+                age = real - float(last) if last != "" else ""
+                writer.writerow(
+                    [
+                        f"{clock():.6f}",
+                        f"{real:.6f}",
+                        name,
+                        *(st.get(key, "") for key in KEYS),
+                        f"{age:.3f}" if age != "" else "",
+                    ]
+                )
+                n += 1
+            k += 1
+    return n