feat(integration-vm): reboot, verify run, failure diagnostics

scripts/integration-vm.py

@@ -261,6 +261,51 @@ def apply(host, certs):
     print(f"applied {host} profile to {name}")
 
 
+def reboot_vm():
+    name, ip, _ = _read_current()
+    sh(["virsh", "reboot", name])
+    time.sleep(5)
+    wait_for_ssh(ip, "ansible")
+    print(f"{name} rebooted, SSH back at {ip}")
+
+
+def run_assert(host, certs):
+    name, ip, _ = _read_current()
+    prof = json.loads(profile_path(host).read_text())
+    write_run_inventory(name, ip, prof["groups"])
+    extra = []
+    for f in prof.get("extra_vars_files", []):
+        extra += ["-e", f"@{INTEG_DIR / f}"]
+    extra += ["-e", f"@{cert_file(certs)}"]
+    cmd = [".venv/bin/ansible-playbook", "-i", str(RUN_DIR) + "/",
+           "tests/integration/verify.yml", "--limit", name] + extra
+    r = sh(cmd, cwd=str(REPO_ROOT), check=False)
+    if r.returncode != 0:
+        dump_diagnostics(name, ip)
+        raise SystemExit(f"VERIFY FAILED for {name} — diagnostics in {DIAG_ROOT}")
+    print(f"VERIFY PASSED for {name}")
+
+
+def dump_diagnostics(name, ip):
+    d = DIAG_ROOT / name
+    d.mkdir(parents=True, exist_ok=True)
+    for label, cmd in [
+        ("nft", "nft list ruleset"),
+        ("docker", "docker ps -a"),
+        ("ss", "ss -tlnp"),
+        ("journal", "journalctl -b --no-pager"),
+        ("critical-chain", "systemd-analyze critical-chain"),
+    ]:
+        r = sh(["ssh", "-o", "StrictHostKeyChecking=no",
+                "-o", "UserKnownHostsFile=/dev/null",
+                f"ansible@{ip}", "sudo " + cmd], check=False, capture=True)
+        (d / f"{label}.txt").write_text((r.stdout or "") + (r.stderr or ""))
+    console = DIAG_ROOT / f"{name}-console.log"
+    if console.exists():
+        shutil.copy(console, d / "console.log")
+    print(f"diagnostics written to {d}", file=sys.stderr)
+
+
 def main(argv=None):
     p = argparse.ArgumentParser(prog="integration-vm", description=__doc__)
     sub = p.add_subparsers(dest="cmd", required=True)