From ab99a9da3eb141d4c3a56a1bae8b50caa2367c97 Mon Sep 17 00:00:00 2001 From: Overlord Date: Tue, 11 Aug 2026 15:43:03 +0200 Subject: [PATCH] Add live integration test suite for `vmic` - Introduced `live_test.sh` to perform end-to-end testing against a real PipeWire session. - Tests cover all major commands (`create`, `edit`, `route`, `list`, `delete`, `wipe`) and validate topology changes, stream migrations, and cleanup behavior. - Ensured safety with guarded destructive operations and thorough resource cleanup mechanisms. - Designed as an opt-in script, not intended for inclusion in automated CI workflows. --- tests/live_test.sh | 368 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100755 tests/live_test.sh diff --git a/tests/live_test.sh b/tests/live_test.sh new file mode 100755 index 0000000..2ff889f --- /dev/null +++ b/tests/live_test.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env bash +# Live integration test suite for vmic. +# +# Exercises every command against a REAL PipeWire session: spawns real +# pw-loopback processes, loads/unloads real pactl loopback modules, and +# moves real sink-inputs/source-outputs. This is deliberately NOT part of +# `cargo test` (see .claude/PROJECT.md: "no integration test suite" was a +# known, deliberate gap - this fills it, but live-only, opt-in, and never +# run automatically). +# +# Usage: +# tests/live_test.sh [options] +# +# Options (env var or flag; flag wins if both given): +# --bin PATH VMIC_BIN vmic binary to test (default: target/debug/vmic) +# --name NAME VMIC_TEST_NAME test vmic name (default: vmictestsuite) +# --input-app APP VMIC_TEST_INPUT_APP `route -i` filter (default: firefox) +# --output-app APP VMIC_TEST_OUTPUT_APP `route -o` filter (default: chromium) +# --hw-source SUBSTR VMIC_TEST_HW_SOURCE `route -s` filter (default: fifine) +# --skip-wipe VMIC_TEST_SKIP_WIPE=1 skip the destructive `wipe` phase +# --no-build skip the `cargo build` preflight +# -h, --help +# +# Safety: +# - Only ever creates/deletes vmics named $NAME, ${NAME}_a, ${NAME}_b. +# - `wipe` tears down EVERY vmic system-wide by design - the wipe phase +# only runs if `vmic list` shows nothing outside those three names at +# that point, or is skipped otherwise (or via --skip-wipe). +# - Moves the given real app streams into/out of the test vmic; an EXIT +# trap always attempts full cleanup (delete test vmics, kill stray +# matching pw-loopback processes, unload stray matching pactl modules) +# even on failure or interruption. +# - Never assumes the user has zero other vmics; all existence/emptiness +# checks are scoped to the test names above. + +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +VMIC_BIN="${VMIC_BIN:-target/debug/vmic}" +NAME="${VMIC_TEST_NAME:-vmictestsuite}" +INPUT_APP="${VMIC_TEST_INPUT_APP:-firefox}" +OUTPUT_APP="${VMIC_TEST_OUTPUT_APP:-chromium}" +HW_SOURCE="${VMIC_TEST_HW_SOURCE:-fifine}" +SKIP_WIPE="${VMIC_TEST_SKIP_WIPE:-0}" +DO_BUILD=1 + +usage() { sed -n '2,/^set -uo/p' "$0" | sed '$d; s/^# \{0,1\}//'; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --bin) VMIC_BIN="$2"; shift 2 ;; + --name) NAME="$2"; shift 2 ;; + --input-app) INPUT_APP="$2"; shift 2 ;; + --output-app) OUTPUT_APP="$2"; shift 2 ;; + --hw-source) HW_SOURCE="$2"; shift 2 ;; + --skip-wipe) SKIP_WIPE=1; shift ;; + --no-build) DO_BUILD=0; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done +NAME="$(tr '[:upper:]' '[:lower:]' <<<"$NAME")" + +RED=$'\e[31m'; GREEN=$'\e[32m'; YELLOW=$'\e[33m'; BLUE=$'\e[34m'; RESET=$'\e[0m' +[[ -t 1 ]] || { RED=""; GREEN=""; YELLOW=""; BLUE=""; RESET=""; } + +PASS=0; FAIL=0; SKIP=0 +section() { echo; echo "${BLUE}== $1 ==${RESET}"; } +pass() { PASS=$((PASS+1)); echo " ${GREEN}PASS${RESET} $1"; } +fail() { FAIL=$((FAIL+1)); echo " ${RED}FAIL${RESET} $1"; [[ -n "${2:-}" ]] && echo " ${2//$'\n'/$'\n '}"; } +skip() { SKIP=$((SKIP+1)); echo " ${YELLOW}SKIP${RESET} $1"; } + +LAST_OUT=""; LAST_CODE=0 +vmic_run() { LAST_OUT="$("$VMIC_BIN" "$@" 2>&1)"; LAST_CODE=$?; } + +expect_exit() { # expect_exit + if [[ "$LAST_CODE" == "$2" ]]; then pass "$1 (exit $LAST_CODE)" + else fail "$1 (expected exit $2, got $LAST_CODE)" "$LAST_OUT"; fi +} +expect_contains() { # expect_contains + if [[ "$LAST_OUT" == *"$2"* ]]; then pass "$1" + else fail "$1 (expected output to contain: $2)" "$LAST_OUT"; fi +} +expect_not_contains() { + if [[ "$LAST_OUT" != *"$2"* ]]; then pass "$1" + else fail "$1 (expected output NOT to contain: $2)" "$LAST_OUT"; fi +} +assert_eq() { if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected '$3', got '$2'"; fi; } +assert_ne() { if [[ "$2" != "$3" ]]; then pass "$1"; else fail "$1" "expected different from '$3', got same '$2'"; fi; } +assert_true() { if "${@:2}" >/dev/null 2>&1; then pass "$1"; else fail "$1"; fi; } + +pw_loopback_count() { # pw_loopback_count + local n; n=$(pgrep -c -f "pw-loopback.*vmic_${1}_" 2>/dev/null); echo "${n:-0}" +} +pulse_id_for() { # pulse_id_for + pactl list short "$1" 2>/dev/null | awk -v n="$2" '$2==n{print $1; exit}' +} +wait_pulse_id() { # wait_pulse_id -> prints id, empty on timeout + local id="" + for _ in $(seq 1 25); do + id="$(pulse_id_for "$1" "$2")" + [[ -n "$id" ]] && { echo "$id"; return 0; } + sleep 0.2 + done + echo "" +} +wait_pw_loopback_count() { # wait_pw_loopback_count + for _ in $(seq 1 25); do + [[ "$(pw_loopback_count "$1")" == "$2" ]] && return 0 + sleep 0.2 + done + return 1 +} +count_sink_inputs_on() { pactl list sink-inputs 2>/dev/null | awk -v id="$1" '$1=="Sink:"{if($2==id)c++} END{print c+0}'; } +count_source_outputs_on() { pactl list source-outputs 2>/dev/null | awk -v id="$1" '$1=="Source:"{if($2==id)c++} END{print c+0}'; } + +cleanup() { + section "Cleanup" + for n in "$NAME" "${NAME}_a" "${NAME}_b"; do + "$VMIC_BIN" delete "$n" >/dev/null 2>&1 || true + done + pkill -f "pw-loopback.*vmic_${NAME}_" 2>/dev/null || true + while read -r modid; do + [[ -n "$modid" ]] && pactl unload-module "$modid" >/dev/null 2>&1 + done < <(pactl list modules 2>/dev/null | awk -v pat="vmic_${NAME}_" ' + /^Module #/{id=$2; sub("#","",id)} /Argument:/{if (index($0, pat)) print id}') + echo " done." + echo + echo "${BLUE}== Results ==${RESET} ${GREEN}$PASS passed${RESET}, ${RED}$FAIL failed${RESET}, ${YELLOW}$SKIP skipped${RESET}" + [[ "$FAIL" -eq 0 ]] +} +trap 'cleanup; exit $(( $? ))' EXIT + +echo "vmic: $VMIC_BIN" +echo "test name: $NAME (+ ${NAME}_a, ${NAME}_b for the wipe phase)" +echo "input app: $INPUT_APP output app: $OUTPUT_APP hw source: $HW_SOURCE" + +if [[ "$DO_BUILD" == "1" ]]; then + section "Build" + if cargo build 2>&1 | tee /dev/stderr | grep -q '^error'; then + echo "build failed, aborting." >&2; exit 1 + fi +fi +[[ -x "$VMIC_BIN" ]] || { echo "binary not found/executable: $VMIC_BIN" >&2; exit 1; } + +# --------------------------------------------------------------------------- +section "Phase 1: CLI surface" +# --------------------------------------------------------------------------- +vmic_run; expect_exit "bare 'vmic' shows help" 2 +expect_contains "bare 'vmic' mentions Usage" "Usage:" +vmic_run -h; expect_exit "'vmic -h'" 0 +vmic_run help; expect_exit "'vmic help'" 0 +vmic_run --version; expect_exit "'vmic --version'" 0 +expect_contains "'--version' mentions vmic" "vmic" +vmic_run create --help; expect_exit "'vmic create --help'" 0 +vmic_run route --help; expect_exit "'vmic route --help'" 0 +vmic_run edit --help; expect_exit "'vmic edit --help'" 0 +expect_contains "'edit --help' documents --loopback-no-mix" "--loopback-no-mix" +for shell in bash zsh fish; do + vmic_run completions "$shell" + assert_eq "'vmic completions $shell' exits 0" "$LAST_CODE" "0" + [[ -n "$LAST_OUT" ]] && pass "'vmic completions $shell' produces output" || fail "'vmic completions $shell' produces output" "(empty)" +done + +# --------------------------------------------------------------------------- +section "Phase 2: error paths (pre-creation)" +# --------------------------------------------------------------------------- +"$VMIC_BIN" delete "$NAME" >/dev/null 2>&1 || true # ensure a clean slate + +vmic_run create; expect_exit "'create' with no name fails" 2 +vmic_run create "bad name!"; assert_true "'create' with an invalid name fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "invalid name error message" "invalid name" +vmic_run route "$NAME" -s x; assert_true "'route' on nonexistent vmic fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "nonexistent vmic error message (route)" "no vmic named" +vmic_run edit "$NAME" -l true; assert_true "'edit' on nonexistent vmic fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "nonexistent vmic error message (edit)" "no vmic named" +vmic_run delete "$NAME"; assert_true "'delete' on nonexistent vmic fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "nonexistent vmic error message (delete)" "no vmic named" + +# --------------------------------------------------------------------------- +section "Phase 3: create - baseline is 2-node" +# --------------------------------------------------------------------------- +vmic_run create "$NAME"; expect_exit "'create $NAME'" 0 +expect_contains "create success message" "Created virtual microphone" +assert_eq "exactly 1 pw-loopback process after create" "$(pw_loopback_count "$NAME")" "1" + +vmic_run list +expect_contains "'list' shows the new vmic" "$NAME" +expect_contains "'list' reports 2-node architecture" "architecture: 2-node (simple)" +expect_contains "'list' reports active status" "status: active" + +vmic_run create "$NAME"; assert_true "'create' twice fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "duplicate create error message" "already exists" +vmic_run route "$NAME"; assert_true "'route' with no flags fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "route nothing-to-do message" "nothing to do" +vmic_run edit "$NAME"; assert_true "'edit' with no flags fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "edit nothing-to-do message" "nothing to do" + +# --------------------------------------------------------------------------- +section "Phase 4: route -i/-o against real streams" +# --------------------------------------------------------------------------- +vmic_run route "$NAME" -i "$INPUT_APP"; expect_exit "'route -i $INPUT_APP'" 0 +if [[ "$LAST_OUT" == *"matched 0 streams"* ]]; then + skip "no active '$INPUT_APP' sink-input right now - CLI path still ran cleanly" +else + expect_contains "'-i $INPUT_APP' reports a move" "-> moved to" + sink_id="$(wait_pulse_id sinks "vmic_${NAME}_sink")" + if [[ -n "$sink_id" ]]; then + n="$(count_sink_inputs_on "$sink_id")" + assert_true "'$INPUT_APP' stream now lands on vmic sink" [ "$n" -ge 1 ] + else + fail "resolve vmic sink's pulse id" "timed out" + fi +fi + +vmic_run route "$NAME" -o "$OUTPUT_APP"; expect_exit "'route -o $OUTPUT_APP'" 0 +if [[ "$LAST_OUT" == *"matched 0 streams"* ]]; then + skip "no active '$OUTPUT_APP' source-output right now - CLI path still ran cleanly" +else + expect_contains "'-o $OUTPUT_APP' reports a move" "-> moved to" + mic_id="$(wait_pulse_id sources "vmic_${NAME}_mic")" + if [[ -n "$mic_id" ]]; then + n="$(count_source_outputs_on "$mic_id")" + assert_true "'$OUTPUT_APP' stream now reads from vmic mic" [ "$n" -ge 1 ] + else + fail "resolve vmic mic's pulse id" "timed out" + fi +fi + +# --------------------------------------------------------------------------- +section "Phase 5: route -s with loopback OFF - stays 2-node" +# --------------------------------------------------------------------------- +sink_before="$(pulse_id_for sinks "vmic_${NAME}_sink")" +vmic_run route "$NAME" -s "$HW_SOURCE"; expect_exit "'route -s $HW_SOURCE'" 0 +expect_contains "mix message notes it's on the sink (no isolation needed)" "sink)" +assert_eq "still 1 pw-loopback process (no upgrade without loopback on)" "$(pw_loopback_count "$NAME")" "1" +vmic_run list +expect_contains "'list' shows the mixed source" "mixed source:" + +vmic_run route "$NAME" -s "$HW_SOURCE"; expect_exit "re-'route -s $HW_SOURCE' (same source, idempotent)" 0 +sink_after="$(pulse_id_for sinks "vmic_${NAME}_sink")" +assert_eq "re-linking the same source doesn't migrate anything" "$sink_after" "$sink_before" + +# --------------------------------------------------------------------------- +section "Phase 6: edit -l true with a source mixed - upgrades to 4-node" +# --------------------------------------------------------------------------- +sink_before="$(pulse_id_for sinks "vmic_${NAME}_sink")" +mic_before="$(pulse_id_for sources "vmic_${NAME}_mic")" +vmic_run edit "$NAME" -l true; expect_exit "'edit -l true'" 0 +assert_true "upgraded to 2 pw-loopback processes" wait_pw_loopback_count "$NAME" 2 +vmic_run list +expect_contains "'list' reports 4-node architecture" "architecture: 4-node (source isolated from self-monitor)" + +sink_id="$(wait_pulse_id sinks "vmic_${NAME}_sink")" +mic_id="$(wait_pulse_id sources "vmic_${NAME}_mic")" +assert_ne "sink node was actually recreated (new pulse id)" "$sink_id" "$sink_before" +assert_ne "mic node was actually recreated (new pulse id)" "$mic_id" "$mic_before" + +if [[ -n "$sink_id" ]]; then + n="$(count_sink_inputs_on "$sink_id")" + if [[ "$n" -ge 1 ]]; then pass "'$INPUT_APP' stream auto-reconnected after upgrade" + else skip "no '$INPUT_APP' stream was active to verify reconnection"; fi +fi +if [[ -n "$mic_id" ]]; then + n="$(count_source_outputs_on "$mic_id")" + if [[ "$n" -ge 1 ]]; then pass "'$OUTPUT_APP' stream auto-reconnected after upgrade" + else skip "no '$OUTPUT_APP' stream was active to verify reconnection"; fi +fi + +# --------------------------------------------------------------------------- +section "Phase 7: edit --loopback-no-mix true - downgrades to 2-node" +# --------------------------------------------------------------------------- +sink_before="$(pulse_id_for sinks "vmic_${NAME}_sink")" +vmic_run edit "$NAME" --loopback-no-mix true; expect_exit "'edit --loopback-no-mix true'" 0 +assert_true "downgraded to 1 pw-loopback process" wait_pw_loopback_count "$NAME" 1 +vmic_run list +expect_contains "'list' reports 2-node + no-mix architecture" "architecture: 2-node (source audible in self-monitor - see --loopback-no-mix)" +sink_after="$(pulse_id_for sinks "vmic_${NAME}_sink")" +assert_ne "sink node was recreated on downgrade" "$sink_after" "$sink_before" + +# --------------------------------------------------------------------------- +section "Phase 8: edit --loopback-no-mix false - re-upgrades to 4-node" +# --------------------------------------------------------------------------- +vmic_run edit "$NAME" --loopback-no-mix false; expect_exit "'edit --loopback-no-mix false'" 0 +assert_true "re-upgraded to 2 pw-loopback processes" wait_pw_loopback_count "$NAME" 2 +vmic_run list +expect_contains "'list' back to 4-node architecture" "architecture: 4-node (source isolated from self-monitor)" + +# --------------------------------------------------------------------------- +section "Phase 9: volumes" +# --------------------------------------------------------------------------- +vmic_run edit "$NAME" -v 55; expect_exit "'edit -v 55'" 0 +vmic_run list; expect_contains "loopback volume shows 55%" "loopback volume: 55%" +vmic_run edit "$NAME" -sv 66; expect_exit "'edit -sv 66'" 0 +vmic_run list; expect_contains "source volume shows 66%" "source volume: 66%" +vmic_run edit "$NAME" -v 0.8; expect_exit "'edit -v 0.8' (fraction form)" 0 +vmic_run list; expect_contains "fraction volume normalizes to 80%" "loopback volume: 80%" +vmic_run edit "$NAME" -v 999; expect_exit "'edit -v 999' still succeeds (clamped)" 0 +expect_contains "over-range volume warns about clamping" "clamped" +vmic_run list; expect_contains "clamped volume shows as 255%" "loopback volume: 255%" + +# --------------------------------------------------------------------------- +section "Phase 10: route -s none - downgrades back to 2-node" +# --------------------------------------------------------------------------- +vmic_run route "$NAME" -s none; expect_exit "'route -s none'" 0 +expect_contains "removal message" "Removed source mix" +assert_true "downgraded to 1 pw-loopback process" wait_pw_loopback_count "$NAME" 1 +vmic_run list; expect_not_contains "'list' no longer shows a mixed source" "mixed source:" + +vmic_run route "$NAME" -s none; expect_exit "re-'route -s none' with nothing mixed" 0 +expect_contains "no-op removal message" "no source is mixed" + +# --------------------------------------------------------------------------- +section "Phase 11: edit -l false with no source mixed - no-op topology" +# --------------------------------------------------------------------------- +sink_before="$(pulse_id_for sinks "vmic_${NAME}_sink")" +vmic_run edit "$NAME" -l false; expect_exit "'edit -l false'" 0 +assert_eq "still 1 pw-loopback process" "$(pw_loopback_count "$NAME")" "1" +sink_after="$(pulse_id_for sinks "vmic_${NAME}_sink")" +assert_eq "sink was NOT recreated (already Simple2Node, no-op)" "$sink_after" "$sink_before" +vmic_run list; expect_contains "'list' shows loopback disabled" "self-monitor loopback: no" + +# --------------------------------------------------------------------------- +section "Phase 12: route -s error paths" +# --------------------------------------------------------------------------- +vmic_run route "$NAME" -s zzz_definitely_not_a_real_source_zzz_12345 +assert_true "'route -s' on a nonexistent source fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "no-matching-source error message" "no matching source" + +# --------------------------------------------------------------------------- +section "Phase 13: delete" +# --------------------------------------------------------------------------- +vmic_run delete "$NAME"; expect_exit "'delete $NAME'" 0 +assert_true "0 pw-loopback processes after delete" wait_pw_loopback_count "$NAME" 0 +vmic_run list; expect_not_contains "'list' no longer mentions $NAME" "$NAME" +vmic_run delete "$NAME"; assert_true "'delete' twice fails" [ "$LAST_CODE" -ne 0 ] +expect_contains "double-delete error message" "no vmic named" + +# --------------------------------------------------------------------------- +section "Phase 14: wipe (guarded - destroys ALL vmics system-wide)" +# --------------------------------------------------------------------------- +if [[ "$SKIP_WIPE" == "1" ]]; then + skip "wipe phase (--skip-wipe passed)" +else + vmic_run list + mapfile -t existing < <(echo "$LAST_OUT" | grep -v '^ ' | grep -v '^$' | grep -v '^No virtual mics created\.$' || true) + other=0 + for n in "${existing[@]:-}"; do + [[ -z "$n" ]] && continue + case "$n" in "${NAME}_a"|"${NAME}_b") ;; *) other=1 ;; esac + done + if [[ "$other" == "1" ]]; then + skip "wipe phase (other, non-test vmics exist: ${existing[*]}) - not safe to run a system-wide wipe" + else + vmic_run create "${NAME}_a"; expect_exit "create throwaway 2-node vmic for wipe test" 0 + vmic_run create "${NAME}_b"; expect_exit "create throwaway vmic" 0 + vmic_run route "${NAME}_b" -s "$HW_SOURCE"; expect_exit "mix a source into it" 0 + vmic_run edit "${NAME}_b" -l true; expect_exit "upgrade it to 4-node" 0 + assert_eq "throwaway_a is 2-node before wipe" "$(pw_loopback_count "${NAME}_a")" "1" + assert_eq "throwaway_b is 4-node before wipe" "$(pw_loopback_count "${NAME}_b")" "2" + + vmic_run wipe; expect_exit "'vmic wipe'" 0 + expect_contains "wipe reports what it did" "Wiped all virtual mics" + assert_true "0 processes left for throwaway_a" wait_pw_loopback_count "${NAME}_a" 0 + assert_true "0 processes left for throwaway_b" wait_pw_loopback_count "${NAME}_b" 0 + vmic_run list; expect_contains "'list' is empty after wipe" "No virtual mics created." + fi +fi