sts-libs 2.3.0__tar.gz → 2.4.0__tar.gz

sts_libs-2.4.0/.scripts/sync_sts.py

@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+# Copyright: Contributors to the sts project
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Sync sts-libs and tests to remote hosts for development.
+
+Builds a wheel from the local tree, installs it via uv on the remote
+host, and rsyncs tests/, plans/ and .fmf/ so you can run::
+
+    ssh root@HOST 'cd /root/sts && pytest tests/ -m ci'
+
+Alternatively, --mr N installs python3-sts-libs from the packit copr
+repo for a merge request (Fedora and RHEL/EPEL 10+ x86_64 only).
+
+Steps within a single host are pipelined with asyncio: the wheel
+builds locally while distro detection, uv/python setup, and test
+syncing proceed on the remote host in parallel.  Multiple hosts
+are synced concurrently.
+
+Examples::
+
+    sync_sts.py 10.0.186.174
+    sync_sts.py 10.0.186.174 my-beaker-host.example.com
+    sync_sts.py -n 10.0.186.174           # tests only, no wheel
+    sync_sts.py --mr 190 10.0.186.174
+"""
+
+# ruff: noqa: T201, D102, D103
+from __future__ import annotations
+
+import argparse
+import asyncio
+import contextlib
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+REMOTE_DIR = '/root/sts'
+_SSH_BASE = ('-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10')
+_ssh_mux: list[str] = []
+
+
+def _ssh_opts() -> tuple[str, ...]:
+    return _SSH_BASE + tuple(_ssh_mux)
+
+
+_RED = '\033[0;31m'
+_GREEN = '\033[0;32m'
+_YELLOW = '\033[1;33m'
+_BLUE = '\033[0;34m'
+_NC = '\033[0m'
+
+
+def info(msg: str, host: str | None = None) -> None:
+    tag = f'[{host}] ' if host else ''
+    print(f'{_BLUE}[INFO]{_NC}  {tag}{msg}')
+
+
+def ok(msg: str, host: str | None = None) -> None:
+    tag = f'[{host}] ' if host else ''
+    print(f'{_GREEN}[OK]{_NC}    {tag}{msg}')
+
+
+def warn(msg: str, host: str | None = None) -> None:
+    tag = f'[{host}] ' if host else ''
+    print(f'{_YELLOW}[WARN]{_NC}  {tag}{msg}')
+
+
+def err(msg: str, host: str | None = None) -> None:
+    tag = f'[{host}] ' if host else ''
+    print(f'{_RED}[ERROR]{_NC} {tag}{msg}', file=sys.stderr)
+
+
+# ── Async subprocess helpers ────────────────────────────────────────
+
+
+async def run_cmd(
+    cmd: list[str],
+    *,
+    check: bool = True,
+    cwd: Path | None = None,
+) -> subprocess.CompletedProcess[str]:
+    """Run a local command asynchronously."""
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        cwd=cwd,
+    )
+    stdout_bytes, stderr_bytes = await proc.communicate()
+    result = subprocess.CompletedProcess(
+        cmd,
+        proc.returncode or 0,
+        stdout_bytes.decode(),
+        stderr_bytes.decode(),
+    )
+    if check and result.returncode != 0:
+        raise subprocess.CalledProcessError(result.returncode, cmd, result.stdout, result.stderr)
+    return result
+
+
+async def ssh(
+    host: str,
+    command: str,
+    *,
+    check: bool = True,
+) -> subprocess.CompletedProcess[str]:
+    return await run_cmd(
+        ['ssh', *_ssh_opts(), f'root@{host}', command],
+        check=check,
+    )
+
+
+async def scp_to(src: Path, host: str, dst: str) -> None:
+    await run_cmd(['scp', *_ssh_opts(), str(src), f'root@{host}:{dst}'])
+
+
+async def rsync_to(src: Path, host: str, dst: str) -> None:
+    await run_cmd(
+        [
+            'rsync',
+            '-az',
+            '--delete',
+            '-e',
+            f'ssh {" ".join(_ssh_opts())}',
+            f'{src}/',
+            f'root@{host}:{dst}/',
+        ]
+    )
+
+
+# ── Distro detection ────────────────────────────────────────────────
+
+
+@dataclass
+class Distro:
+    """Linux distribution detected on the remote host."""
+
+    id: str
+    version_id: str
+
+    @property
+    def major(self) -> int:
+        return int(self.version_id.split('.')[0])
+
+    @property
+    def has_modern_python(self) -> bool:
+        """System python3 is >= 3.12 (Fedora, RHEL/CentOS 10+)."""
+        if self.id == 'fedora':
+            return True
+        return self.id in ('rhel', 'centos') and self.major >= 10
+
+    @property
+    def needs_uv_python(self) -> bool:
+        """No python3.12 dnf package — uv must install it (RHEL 8.8, 9.2)."""
+        return self.id == 'rhel' and self.version_id in ('8.8', '9.2')
+
+    @property
+    def python_bin(self) -> str:
+        """Python binary name for shell commands."""
+        return 'python3' if self.has_modern_python else 'python3.12'
+
+    @property
+    def copr_chroot(self) -> str | None:
+        """Copr chroot name, or None if copr builds are unavailable."""
+        if self.id == 'fedora':
+            return f'fedora-{self.version_id}-x86_64'
+        if self.id in ('rhel', 'centos') and self.major >= 10:
+            return f'epel-{self.major}-x86_64'
+        return None
+
+    def __str__(self) -> str:
+        return f'{self.id}-{self.version_id}'
+
+
+async def detect_distro(host: str) -> Distro:
+    """Parse /etc/os-release on the remote host."""
+    result = await ssh(host, 'cat /etc/os-release', check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f'Cannot read /etc/os-release on {host}')
+
+    props: dict[str, str] = {}
+    for line in result.stdout.splitlines():
+        if '=' in line:
+            key, _, value = line.partition('=')
+            props[key] = value.strip('"')
+
+    return Distro(
+        id=props.get('ID', 'unknown'),
+        version_id=props.get('VERSION_ID', '0'),
+    )
+
+
+# ── Wheel build ─────────────────────────────────────────────────────
+
+
+async def _warn_untracked() -> None:
+    """Warn if sts_libs/src has untracked files."""
+    result = await run_cmd(
+        ['git', 'ls-files', '--others', '--exclude-standard', 'sts_libs/src/'],
+        cwd=REPO_ROOT,
+        check=False,
+    )
+    if result.stdout.strip():
+        warn('Untracked files in sts_libs/src/ — they may end up in the wheel')
+
+
+async def build_wheel() -> Path:
+    """Build an sts-libs wheel and return its path."""
+    await _warn_untracked()
+    info('Building wheel')
+    await run_cmd(
+        ['uv', 'build', '--wheel', '--out-dir', 'dist/'],
+        cwd=REPO_ROOT,
+    )
+    wheels = sorted((REPO_ROOT / 'dist').glob('sts_libs-*-py3-none-any.whl'))
+    if not wheels:
+        err(f'No wheel found in {REPO_ROOT / "dist"} after build')
+        sys.exit(1)
+    ok(f'Built {wheels[-1].name}')
+    return wheels[-1]
+
+
+# ── Wheel install ────────────────────────────────────────────────────
+
+
+async def setup_uv_and_python(host: str, distro: Distro) -> None:
+    """Ensure uv and Python >= 3.12 are available on the remote host."""
+    steps: list[str] = []
+
+    if distro.has_modern_python:
+        steps.append(
+            'command -v uv >/dev/null 2>&1'
+            ' || dnf install -y uv 2>/dev/null'
+            ' || (dnf install -y python3-pip && pip3 install uv)',
+        )
+    elif distro.needs_uv_python and distro.major == 8:
+        steps.extend(
+            [
+                'dnf install -y python3.11-pip',
+                'python3.11 -m pip install uv',
+                'uv python install 3.12',
+            ]
+        )
+    elif distro.needs_uv_python:
+        steps.extend(
+            [
+                'dnf install -y python3-pip',
+                'python3 -m pip install uv',
+                'uv python install 3.12',
+            ]
+        )
+    else:
+        steps.extend(
+            [
+                'dnf install -y python3.12-pip',
+                'python3.12 -m pip install uv',
+            ]
+        )
+
+    if not distro.has_modern_python:
+        steps.append("dnf remove -y 'python3*-pytest' 2>/dev/null || true")
+
+    info('Setting up Python environment', host=host)
+    await ssh(host, ' && '.join(steps))
+
+
+async def ensure_pytest_on_path(host: str) -> None:
+    """Symlink pytest to /usr/local/bin if it is not already on PATH."""
+    await ssh(
+        host,
+        'command -v pytest >/dev/null 2>&1 && exit 0; '
+        'for d in /usr/local/bin /root/.local/bin; do '
+        '  [ -f "$d/pytest" ] && ln -sf "$d/pytest" /usr/local/bin/pytest && exit 0; '
+        'done; '
+        'py=$(uv python find 3.12 2>/dev/null) && '
+        '  [ -f "$(dirname "$py")/pytest" ] && '
+        '  ln -sf "$(dirname "$py")/pytest" /usr/local/bin/pytest; '
+        'true',
+        check=False,
+    )
+
+
+async def copy_and_install_wheel(host: str, distro: Distro, wheel: Path) -> None:
+    """Copy the wheel to the host and install via uv pip.
+
+    Assumes setup_uv_and_python has already run.
+    """
+    remote_wheel = f'/tmp/{wheel.name}'
+
+    info(f'Copying {wheel.name}', host=host)
+    await scp_to(wheel, host, remote_wheel)
+
+    info('Installing sts-libs', host=host)
+    if distro.has_modern_python:
+        await ssh(host, f'uv pip install --system {remote_wheel} pytest fmf')
+    elif distro.needs_uv_python:
+        await ssh(
+            host,
+            f'uv tool install pytest --with {remote_wheel} --with fmf --python 3.12',
+        )
+        await ensure_pytest_on_path(host)
+    else:
+        await ssh(host, f'uv pip install --system --python 3.12 {remote_wheel} pytest fmf')
+        await ensure_pytest_on_path(host)
+
+    await ssh(host, f'rm -f {remote_wheel}')
+
+
+# ── Copr MR mode ─────────────────────────────────────────────────────
+
+
+async def install_copr(host: str, distro: Distro, mr: int) -> None:
+    """Enable the packit copr repo for a merge request and dnf-install."""
+    chroot = distro.copr_chroot
+    if chroot is None:
+        err(f'Copr MR install not supported on {distro}', host=host)
+        err('Only Fedora and RHEL/EPEL 10+ (x86_64) have copr builds', host=host)
+        raise SystemExit(1)
+
+    copr_repo = f'packit/gitlab.com-rh-kernel-stqe-sts-{mr}'
+    info(f'Enabling copr: {copr_repo} ({chroot})', host=host)
+
+    await ssh(
+        host,
+        ' && '.join(
+            [
+                "dnf install -y 'dnf*-command(copr)'",
+                f'dnf copr enable -y {copr_repo} {chroot}',
+                'dnf install -y python3-sts-libs python3-pytest',
+            ]
+        ),
+    )
+    ok(f'Installed python3-sts-libs from MR #{mr}', host=host)
+
+
+# ── Common ───────────────────────────────────────────────────────────
+
+
+async def sync_tests(host: str) -> None:
+    """Sync tests, plans, and .fmf metadata to the remote host."""
+    info('Syncing tests, plans, .fmf', host=host)
+    await ssh(host, f'mkdir -p {REMOTE_DIR}', check=False)
+    await asyncio.gather(
+        rsync_to(REPO_ROOT / 'tests', host, f'{REMOTE_DIR}/tests'),
+        rsync_to(REPO_ROOT / 'plans', host, f'{REMOTE_DIR}/plans'),
+        rsync_to(REPO_ROOT / '.fmf', host, f'{REMOTE_DIR}/.fmf'),
+    )
+
+
+async def verify(host: str, distro: Distro) -> None:
+    """Print versions and test file count as a sanity check."""
+    python = distro.python_bin
+    info('Verifying', host=host)
+    result = await ssh(
+        host,
+        (
+            f'set -e; cd {REMOTE_DIR}; '
+            f'echo "  python:  $({python} --version 2>&1)"; '
+            f'echo "  pytest:  $(pytest --version 2>&1 | head -1)"; '
+            f"echo \"  sts:     $({python} -c 'import sts; print(sts.__version__)' 2>/dev/null"
+            f" || echo 'installed (pytest env)')\"; "
+            f'echo "  tests:   $(find tests -name \'test_*.py\' | wc -l) test files"'
+        ),
+    )
+    for line in result.stdout.splitlines():
+        print(f'  {line}')
+
+
+async def sync_one(
+    host: str,
+    *,
+    mr: int | None = None,
+    wheel_task: asyncio.Task[Path] | None = None,
+) -> bool:
+    """Sync to a single host. Returns True on success.
+
+    Pipeline within a single host::
+
+        build_wheel (local) ──────────────────────────────┐
+        │ connectivity ─→ detect_distro ─→ setup_uv ──────┤
+        │                 │ sync tests/plans/.fmf ─────────┤
+        └─→ scp + install wheel ───────→                   │
+                                          verify ──→ done
+    """
+    try:
+        if (await ssh(host, 'true', check=False)).returncode != 0:
+            err(f'Cannot reach {host} — skipping', host=host)
+            return False
+
+        # Start syncing tests immediately — no dependency on distro or wheel
+        sync_task = asyncio.create_task(sync_tests(host))
+
+        distro = await detect_distro(host)
+        info(f'══ {distro} @ {host} ══')
+
+        if mr is not None:
+            await asyncio.gather(install_copr(host, distro, mr), sync_task)
+        elif wheel_task is not None:
+            # Run uv/python setup while wheel builds locally
+            _, wheel = await asyncio.gather(
+                setup_uv_and_python(host, distro),
+                wheel_task,
+            )
+            await asyncio.gather(
+                copy_and_install_wheel(host, distro, wheel),
+                sync_task,
+            )
+        else:
+            await sync_task
+
+        await verify(host, distro)
+
+    except subprocess.CalledProcessError as exc:
+        err(f'Command failed on {host}: {exc.cmd}', host=host)
+        return False
+    except RuntimeError as exc:
+        err(str(exc), host=host)
+        return False
+    else:
+        ok(f'Done: {distro} @ {host}', host=host)
+        info(f"  ssh root@{host} 'cd {REMOTE_DIR} && pytest tests/ -m ci'")
+        return True
+
+
+# ── CLI ──────────────────────────────────────────────────────────────
+
+
+async def async_main(args: argparse.Namespace) -> int:
+    """Async entry point — fans out to all hosts concurrently."""
+    ctrl_dir = tempfile.mkdtemp(prefix='sts-sync-')
+    _ssh_mux[:] = [
+        '-o',
+        'ControlMaster=auto',
+        '-o',
+        f'ControlPath={ctrl_dir}/%C',
+        '-o',
+        'ControlPersist=10',
+    ]
+    try:
+        wheel_task: asyncio.Task[Path] | None = None
+        if not args.mr and not args.no_lib:
+            wheel_task = asyncio.create_task(build_wheel())
+
+        results = await asyncio.gather(
+            *(sync_one(host, mr=args.mr, wheel_task=wheel_task) for host in args.hosts),
+        )
+
+        if wheel_task is not None and not wheel_task.done():
+            wheel_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await wheel_task
+
+        print('════════════════════════════════════════════')
+        total = len(args.hosts)
+        failed = sum(not r for r in results)
+        if failed == 0:
+            ok(f'All {total} host{"s" if total > 1 else ""} synced successfully')
+            return 0
+        warn(f'{failed}/{total} hosts failed')
+        return 1
+    finally:
+        _ssh_mux.clear()
+        shutil.rmtree(ctrl_dir, ignore_errors=True)
+
+
+def main() -> int:
+    """Entry point."""
+    parser = argparse.ArgumentParser(
+        description='Sync sts-libs and tests to remote hosts for development',
+        epilog=(
+            'examples:\n'
+            '  %(prog)s 10.0.186.174\n'
+            '  %(prog)s 10.0.186.174 my-beaker-host.example.com\n'
+            '  %(prog)s -n 10.0.186.174           # tests only\n'
+            '  %(prog)s --mr 190 10.0.186.174\n'
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument('hosts', nargs='+', metavar='HOST', help='IP addresses or hostnames')
+    parser.add_argument('--mr', type=int, metavar='N', help='install from copr MR repo instead of local wheel')
+    parser.add_argument('-n', '--no-lib', action='store_true', help='sync tests only, skip wheel build and install')
+
+    args = parser.parse_args()
+    return asyncio.run(async_main(args))
+
+
+if __name__ == '__main__':
+    sys.exit(main())

{sts_libs-2.3.0 → sts_libs-2.4.0}/AGENTS.md

@@ -85,7 +85,7 @@ Every source file uses `from __future__ import annotations`. Types needed only f
 
 ### Docstrings
 
-Google-style. Use `Attributes:` (not `Args:`) for Pydantic model class fields — griffe maps `Args:` to function signatures and will warn. Use `Args:` for function/method parameters. Don't repeat type annotations in docstrings.
+Keep docstrings lean. Remove `Attributes:` sections from Pydantic models — field names and type annotations are self-documenting. Remove `Args:` sections where parameter names and types are obvious (e.g. `device: str | Path`). Only keep `Args:` when documenting non-obvious valid values (e.g. listing valid policy strings for a `str` parameter). Remove trivial `Example:` blocks. One-line docstrings are preferred for methods. Don't repeat type annotations in docstrings.
 
 ### Typed Kwargs
 
@@ -104,6 +104,12 @@ Use `ClassVar` for constants that should not become Pydantic fields (e.g., `DEV_
 - **Unit tests** (`sts_libs/tests/`) — mock-based, patch `subprocess.run`. Run with `uv run pytest`. If tests are skipped or fail due to environment issues, use `dev-container/run.sh test` (may need `dev-container/build.sh` first).
 - **Storage tests** (`tests/`) — actual Linux storage tests, require real hardware/VMs and root. Run via tmt, not in CI.
 - Tests marked `@pytest.mark.destructive` are auto-skipped without root.
+- **Package installation** — use `ensure_installed()` in fixtures rather than tmt `require:` blocks. This lets tests run directly with pytest without tmt.
+
+### TMT / FMF Conventions
+
+- `component:` should match the subsystem name (e.g. `ahci`, `scsi`, `lvm`). Never use `component: kernel`.
+- Don't add `require:` for packages that fixtures install via `ensure_installed()`.
 
 ## Tools
 

{sts_libs-2.3.0 → sts_libs-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sts-libs
-Version: 2.3.0
+Version: 2.4.0
 Summary: Library for pytest-based Linux storage tests
 Project-URL: Repository, https://gitlab.com/rh-kernel-stqe/sts/
 Author-email: Martin Hoyer <mhoyer@redhat.com>, Filip Suba <fsuba@redhat.com>, Zhaojuan Guo <zguo@redhat.com>, Bruno Goncalves <bgoncalv@redhat.com>, Jakub Krysl <jkrysl@redhat.com>

sts_libs-2.4.0/plans/ahci/main.fmf

@@ -0,0 +1,9 @@
+summary: AHCI SATA storage test suite
+discover:
+    how: fmf
+    filter: tag:ahci
+provision:
+    how: beaker
+    hardware:
+        device:
+            driver: ahci

sts_libs-2.4.0/plans/block/virtual-hw.fmf

@@ -0,0 +1,16 @@
+summary: Block tests requiring multiple CPUs and elevated memory
+description: >
+  Virtual block-device tests that need stronger hardware than the
+  baseline plan — CPU hotplug stress, cgroup throttling, and similar
+  workloads that require multiple processors or extra memory.
+discover:
+  how: fmf
+  filter:
+    - "tag:block & (tag:hw-cpu | tag:hw-memory)"
+provision:
+  hardware:
+    cpu:
+      processors: '>= 4'
+    disk:
+      - size: '>= 20 GB'
+    memory: '>= 8 GB'

sts_libs-2.4.0/plans/block/virtual.fmf

@@ -0,0 +1,13 @@
+summary: Virtual block-layer regression tests
+description: >
+  Block-layer coverage using only software-backed virtual devices
+  (brd, nbd, null_blk, scsi_debug, zram, loop, md). Tests run on an
+  ordinary VM without special hardware.
+discover:
+  how: fmf
+  filter:
+    - "tag:block & !tag:hw-cpu & !tag:hw-memory"
+provision:
+  hardware:
+    disk:
+      - size: '>= 20 GB'

sts_libs-2.4.0/plans/scsi/main.fmf

@@ -0,0 +1,4 @@
+summary: Generic SCSI block device tests
+discover:
+    how: fmf
+    filter: tag:scsi

{sts_libs-2.3.0 → sts_libs-2.4.0}/plans/sts-ci/ci.fmf

@@ -12,4 +12,5 @@ discover:
       - "tag: -bootc"
       - "tag: -hw-memory"
       - "tag: -hw-disk"
+      - "tag: -hw-cpu"
       - "tier: 1"

{sts_libs-2.3.0 → sts_libs-2.4.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = 'sts-libs'
 description = 'Library for pytest-based Linux storage tests'
 readme = 'sts_libs/README.md'
 requires-python = '>=3.12'
-version = "2.3.0"
+version = "2.4.0"
 authors = [
   {name = 'Martin Hoyer', email = 'mhoyer@redhat.com'},
   {name = 'Filip Suba', email = 'fsuba@redhat.com'},
@@ -230,6 +230,8 @@ directory = 'coverage_html'
 [tool.basedpyright]
 pythonVersion = "3.12"
 pythonPlatform = "Linux"
+venvPath = "."
+venv = ".venv"
 include = ["sts_libs/src/sts", "tests"]
 exclude = ["sts_libs/tests"]
 typeCheckingMode = "standard"