test: migrate examples to XXXX-category-name layout + split expected streams
Rename all example tests/companions to the XXXX-category-test-name scheme (per-category 100-blocks: basic 0010, types 0100, ... errors 1000, diagnostics 1100, ffi 1200, ffi-objc 1300, ffi-jni 1400, vectors 1500, platform 1600). Companions and dir/C fixtures move in lockstep with their parent test; #import/#source/#include paths rewritten to match. Expected output now lives in examples/expected/ (a sibling dir of the tests) split into three streams per the new convention: <name>.exit / <name>.stdout / <name>.stderr (+ optional <name>.ir) run_examples.sh rewritten: scans examples/ and issues/ for an expected/<name>.exit marker, captures stdout and stderr separately (no more 2>&1), compares each stream + exit + optional IR snapshot. Behavior validated unchanged: every renamed test reproduces its prior merged output + exit (diffs limited to file paths/basenames embedded in diagnostics + traces, which correctly reflect the new names). Suite: 292 passed, 0 failed. 50-smoke.sx split + issue relocation + docs follow in subsequent commits.
This commit is contained in:
agra
@@ -1,14 +1,24 @@
|
||||
#!/bin/bash
|
||||
# Example regression test runner
|
||||
# Example/issue regression test runner.
|
||||
# Usage: ./tests/run_examples.sh [--update]
|
||||
# --update: regenerate expected output and exit code files
|
||||
# --update: regenerate expected output (.exit/.stdout/.stderr, and .ir where present)
|
||||
#
|
||||
# Layout (per CLAUDE.md): expected output lives in an `expected/` dir that
|
||||
# sits NEXT TO the test file, with three streams split out:
|
||||
# <root>/<name>.sx
|
||||
# <root>/expected/<name>.exit # process exit code
|
||||
# <root>/expected/<name>.stdout # normalized stdout
|
||||
# <root>/expected/<name>.stderr # normalized stderr
|
||||
# <root>/expected/<name>.ir # optional `sx ir` snapshot
|
||||
# A test is any <name>.sx that has an <root>/expected/<name>.exit marker.
|
||||
# Roots scanned: examples/ and issues/.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SX="$ROOT_DIR/zig-out/bin/sx"
|
||||
EXPECTED_DIR="$SCRIPT_DIR/expected"
|
||||
ROOTS=("$ROOT_DIR/examples" "$ROOT_DIR/issues")
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
@@ -25,11 +35,8 @@ normalize() {
|
||||
sed 's/0x[0-9a-f]\{4,\}/0xADDR/g'
|
||||
}
|
||||
|
||||
# Normalize `sx ir` output for snapshot diffing. Strips host-specific
|
||||
# noise so a snapshot taken on one macOS machine matches another:
|
||||
# - target triple / datalayout / module-id headers
|
||||
# - function-attribute groups (target-cpu, frame-pointer string)
|
||||
# - LLVM's auto-suffixed temporary names (%add1, %icmp29 → %add, %icmp)
|
||||
# Normalize `sx ir` output for snapshot diffing (host-specific noise + LLVM
|
||||
# auto-suffixed temporaries).
|
||||
normalize_ir() {
|
||||
sed -E \
|
||||
-e '/^; ModuleID =/d' \
|
||||
@@ -40,92 +47,76 @@ normalize_ir() {
|
||||
-e 's/%([a-z]+)[0-9]+/%\1N/g'
|
||||
}
|
||||
|
||||
for expected_file in "$EXPECTED_DIR"/*.txt; do
|
||||
name=$(basename "$expected_file" .txt)
|
||||
sx_file="$ROOT_DIR/examples/${name}.sx"
|
||||
exit_file="$EXPECTED_DIR/${name}.exit"
|
||||
TMP_ERR="$(mktemp)"
|
||||
trap 'rm -f "$TMP_ERR"' EXIT
|
||||
|
||||
if [[ ! -f "$sx_file" ]]; then
|
||||
SKIP=$((SKIP + 1))
|
||||
continue
|
||||
fi
|
||||
for root in "${ROOTS[@]}"; do
|
||||
expected_dir="$root/expected"
|
||||
[[ -d "$expected_dir" ]] || continue
|
||||
for exit_file in "$expected_dir"/*.exit; do
|
||||
[[ -e "$exit_file" ]] || continue
|
||||
name=$(basename "$exit_file" .exit)
|
||||
sx_file="$root/${name}.sx"
|
||||
out_file="$expected_dir/${name}.stdout"
|
||||
err_file="$expected_dir/${name}.stderr"
|
||||
ir_file="$expected_dir/${name}.ir"
|
||||
|
||||
printf " %-30s" "$name"
|
||||
actual=$(timeout "$TIMEOUT" "$SX" run "$sx_file" 2>&1 | normalize)
|
||||
actual_exit=${PIPESTATUS[0]}
|
||||
|
||||
if [[ $actual_exit -eq 124 ]]; then
|
||||
TIMEOUT_COUNT=$((TIMEOUT_COUNT + 1))
|
||||
echo "TIMEOUT (>${TIMEOUT}s)"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Optional IR-shape snapshot. When tests/expected/<name>.ir exists,
|
||||
# also diff `sx ir <file>` against it. Used to lock down lowering
|
||||
# changes that don't show up in runtime output (e.g., selector
|
||||
# interning trims sel_registerName calls without changing what
|
||||
# the program prints).
|
||||
ir_file="$EXPECTED_DIR/${name}.ir"
|
||||
actual_ir=""
|
||||
has_ir_snapshot=false
|
||||
if [[ -f "$ir_file" || ( $UPDATE -eq 1 && -f "$EXPECTED_DIR/${name}.ir" ) ]]; then
|
||||
has_ir_snapshot=true
|
||||
fi
|
||||
if $has_ir_snapshot; then
|
||||
actual_ir=$("$SX" ir "$sx_file" 2>&1 | normalize_ir)
|
||||
fi
|
||||
|
||||
if [[ $UPDATE -eq 1 ]]; then
|
||||
echo "$actual" > "$expected_file"
|
||||
echo "$actual_exit" > "$exit_file"
|
||||
if $has_ir_snapshot; then
|
||||
echo "$actual_ir" > "$ir_file"
|
||||
if [[ ! -f "$sx_file" ]]; then
|
||||
SKIP=$((SKIP + 1))
|
||||
continue
|
||||
fi
|
||||
echo " updated $name (exit=$actual_exit)"
|
||||
continue
|
||||
fi
|
||||
|
||||
expected=$(cat "$expected_file" | normalize)
|
||||
expected_exit=0
|
||||
if [[ -f "$exit_file" ]]; then
|
||||
printf " %-48s" "$name"
|
||||
actual_out=$(timeout "$TIMEOUT" "$SX" run "$sx_file" 2>"$TMP_ERR" | normalize)
|
||||
actual_exit=${PIPESTATUS[0]}
|
||||
actual_err=$(normalize < "$TMP_ERR")
|
||||
|
||||
if [[ $actual_exit -eq 124 ]]; then
|
||||
TIMEOUT_COUNT=$((TIMEOUT_COUNT + 1))
|
||||
echo "TIMEOUT (>${TIMEOUT}s)"
|
||||
continue
|
||||
fi
|
||||
|
||||
has_ir=false
|
||||
[[ -f "$ir_file" ]] && has_ir=true
|
||||
actual_ir=""
|
||||
if $has_ir; then
|
||||
actual_ir=$("$SX" ir "$sx_file" 2>&1 | normalize_ir)
|
||||
fi
|
||||
|
||||
if [[ $UPDATE -eq 1 ]]; then
|
||||
echo "$actual_out" > "$out_file"
|
||||
echo "$actual_err" > "$err_file"
|
||||
echo "$actual_exit" > "$exit_file"
|
||||
$has_ir && echo "$actual_ir" > "$ir_file"
|
||||
echo "updated (exit=$actual_exit)"
|
||||
continue
|
||||
fi
|
||||
|
||||
expected_out=$(normalize < "$out_file" 2>/dev/null)
|
||||
expected_err=$(normalize < "$err_file" 2>/dev/null)
|
||||
expected_exit=$(cat "$exit_file")
|
||||
fi
|
||||
expected_ir=""
|
||||
if $has_ir_snapshot; then
|
||||
expected_ir=$(cat "$ir_file" | normalize_ir)
|
||||
fi
|
||||
expected_ir=""
|
||||
$has_ir && expected_ir=$(normalize_ir < "$ir_file")
|
||||
|
||||
output_ok=true
|
||||
exit_ok=true
|
||||
ir_ok=true
|
||||
out_ok=true; err_ok=true; exit_ok=true; ir_ok=true
|
||||
[[ "$actual_out" == "$expected_out" ]] || out_ok=false
|
||||
[[ "$actual_err" == "$expected_err" ]] || err_ok=false
|
||||
[[ "$actual_exit" == "$expected_exit" ]] || exit_ok=false
|
||||
if $has_ir && [[ "$actual_ir" != "$expected_ir" ]]; then ir_ok=false; fi
|
||||
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
output_ok=false
|
||||
fi
|
||||
if [[ "$actual_exit" != "$expected_exit" ]]; then
|
||||
exit_ok=false
|
||||
fi
|
||||
if $has_ir_snapshot && [[ "$actual_ir" != "$expected_ir" ]]; then
|
||||
ir_ok=false
|
||||
fi
|
||||
|
||||
if $output_ok && $exit_ok && $ir_ok; then
|
||||
PASS=$((PASS + 1))
|
||||
echo "ok"
|
||||
else
|
||||
FAIL=$((FAIL + 1))
|
||||
echo "FAIL"
|
||||
if ! $output_ok; then
|
||||
diff <(echo "$expected") <(echo "$actual") || true
|
||||
if $out_ok && $err_ok && $exit_ok && $ir_ok; then
|
||||
PASS=$((PASS + 1))
|
||||
echo "ok"
|
||||
else
|
||||
FAIL=$((FAIL + 1))
|
||||
echo "FAIL"
|
||||
$out_ok || { echo " --- stdout diff ---"; diff <(echo "$expected_out") <(echo "$actual_out") || true; }
|
||||
$err_ok || { echo " --- stderr diff ---"; diff <(echo "$expected_err") <(echo "$actual_err") || true; }
|
||||
$exit_ok || echo " exit code: expected=$expected_exit actual=$actual_exit"
|
||||
$ir_ok || { echo " --- IR diff ---"; diff <(echo "$expected_ir") <(echo "$actual_ir") || true; }
|
||||
fi
|
||||
if ! $exit_ok; then
|
||||
echo " exit code: expected=$expected_exit actual=$actual_exit"
|
||||
fi
|
||||
if ! $ir_ok; then
|
||||
echo " IR diff:"
|
||||
diff <(echo "$expected_ir") <(echo "$actual_ir") || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [[ $UPDATE -eq 1 ]]; then
|
||||
|
||||
Reference in New Issue
Block a user