Add regression test harness and baseline metrics
This commit is contained in:
parent
9dbd2b5a7a
commit
ebcf7c227f
2 changed files with 75 additions and 0 deletions
14
docs/regression-baseline.txt
Normal file
14
docs/regression-baseline.txt
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
Generated file count: 932
|
||||
Total build errors: 733578
|
||||
Cannot find errors (dependency, ignored): 268626
|
||||
Fixable build errors: 464952
|
||||
Lint errors: 0
|
||||
|
||||
Note: The current baseline shows a regression. The dominant error
|
||||
(57,784 occurrences) is "multiple incompatible access-level modifiers
|
||||
specified" coming from 31 files that contain `public open class`.
|
||||
In Swift 6.3, `public` and `open` cannot be combined — `open` is
|
||||
itself an access level modifier. The regression was introduced by
|
||||
commit ffe6e10 "Force open class for classes with same-namespace
|
||||
subclasses" which added `open class` but kept the unconditional
|
||||
`public ` prefix in CodeGen.swift.
|
||||
61
scripts/regression-test.sh
Executable file
61
scripts/regression-test.sh
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env bash
|
||||
# Regression test for swift-gtk-gen output.
|
||||
# Run this after any change to CodeGen.swift to catch regressions.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
TEST_DIR="/tmp/regression-test-$$"
|
||||
echo "=== swift-gtk-gen regression test ==="
|
||||
echo "Test output directory: $TEST_DIR"
|
||||
|
||||
# Generate
|
||||
rm -rf "$TEST_DIR"
|
||||
mkdir -p "$TEST_DIR"
|
||||
.build/x86_64-unknown-linux-gnu/debug/swift-gtk-gen \
|
||||
--gir-file /usr/share/gir-1.0/Gtk-4.0.gir \
|
||||
--output "$TEST_DIR" \
|
||||
--generate-all 2>/dev/null
|
||||
|
||||
# Run unit tests
|
||||
echo ""
|
||||
echo "=== Unit tests ==="
|
||||
swift test 2>&1 | tail -2
|
||||
|
||||
# Count errors
|
||||
echo ""
|
||||
echo "=== Generated output metrics ==="
|
||||
file_count=$(find "$TEST_DIR" -name "*.swift" | wc -l)
|
||||
echo "Generated Swift files: $file_count"
|
||||
|
||||
cd "$TEST_DIR"
|
||||
if swift build 2>&1 > /tmp/build-output.log; then
|
||||
echo "Generated code BUILD: PASS"
|
||||
else
|
||||
cannot_find=$(grep "error:" /tmp/build-output.log | grep -c "cannot find" || echo 0)
|
||||
total=$(grep -c "error:" /tmp/build-output.log || echo 0)
|
||||
fixable=$((total - cannot_find))
|
||||
echo "Generated code BUILD: FAIL ($total total errors, $cannot_find dependency-ignored, $fixable fixable)"
|
||||
if [ "$fixable" -gt 0 ]; then
|
||||
echo "First 20 fixable errors:"
|
||||
grep "error:" /tmp/build-output.log | grep -v "cannot find" | head -20
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Lint
|
||||
echo ""
|
||||
echo "=== Lint ==="
|
||||
cd -
|
||||
lint_errors=$(swift format lint --configuration .swift-format --recursive "$TEST_DIR" 2>&1 | grep -c "error:" || echo 0)
|
||||
echo "Lint errors: $lint_errors"
|
||||
if [ "$lint_errors" -gt 0 ]; then
|
||||
echo "First 10 lint errors:"
|
||||
swift format lint --configuration .swift-format --recursive "$TEST_DIR" 2>&1 | grep "error:" | head -10
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm -rf "$TEST_DIR"
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
Loading…
Add table
Add a link
Reference in a new issue