From 64e65bca928882cc5f7d084119b7946c45b7052d Mon Sep 17 00:00:00 2001 From: Patrick McCarty Date: Sun, 10 Apr 2016 20:58:17 -0700 Subject: [PATCH] Run functional tests in separate directories To more easily implement tests, analyze test results, etc, store each test in a separate directory and dump all logs and the web-dir content there as well. The test suite can be run with 'make check', as before, but now tests can be run individually by directly invoking the appropriate test.bats file (at toplevel, or within the test directory). Signed-off-by: Patrick McCarty --- .gitignore | 6 ++- Makefile.am | 20 ++++----- .../{basic.bats => basic/test.bats} | 14 +++--- .../test.bats} | 18 +++----- .../test.bats} | 30 ++++++------- .../{full-run.bats => full-run/test.bats} | 18 +++----- .../{fullfiles.bats => fullfiles/test.bats} | 8 ++-- .../test.bats} | 44 +++++++++---------- test/functional/{pack.bats => pack/test.bats} | 12 ++--- test/functional/swupdlib.bash | 26 ++++++++--- .../{update.bats => update/test.bats} | 8 ++-- 11 files changed, 106 insertions(+), 98 deletions(-) rename test/functional/{basic.bats => basic/test.bats} (57%) rename test/functional/{delete-no-version-bump.bats => delete-no-version-bump/test.bats} (71%) rename test/functional/{full-run-delta.bats => full-run-delta/test.bats} (79%) rename test/functional/{full-run.bats => full-run/test.bats} (79%) rename test/functional/{fullfiles.bats => fullfiles/test.bats} (75%) rename test/functional/{include-version-bump.bats => include-version-bump/test.bats} (69%) rename test/functional/{pack.bats => pack/test.bats} (66%) rename test/functional/{update.bats => update/test.bats} (77%) diff --git a/.gitignore b/.gitignore index 0b8881b..d26438f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,9 @@ coverage/ src/*.gcno src/*.gcda tap-driver.sh -test/functional/*.log -test/functional/*.trs +test/functional/*/*.log +test/functional/*/*.trs +test/functional/*/logs/ +test/functional/*/web-dir/ swupd-*.log test-suite.log diff --git a/Makefile.am b/Makefile.am index 212a873..76d4404 100644 --- a/Makefile.am +++ b/Makefile.am @@ -96,26 +96,26 @@ noinst_HEADERS = \ include/swupd.h \ include/xattrs.h -TEST_EXTENSIONS = .sh +TEST_EXTENSIONS = .bats if ENABLE_TESTS tap_driver = env AM_TAP_AWK='$(AWK)' $(SHELL) \ $(top_srcdir)/tap-driver.sh LOG_DRIVER = $(tap_driver) -SH_LOG_DRIVER = $(tap_driver) +BATS_LOG_DRIVER = $(tap_driver) TESTS = $(dist_check_SCRIPTS) dist_check_SCRIPTS = \ - test/functional/basic.bats \ - test/functional/delete-no-version-bump.bats \ - test/functional/include-version-bump.bats \ - test/functional/update.bats \ - test/functional/fullfiles.bats \ - test/functional/pack.bats \ - test/functional/full-run.bats \ - test/functional/full-run-delta.bats + test/functional/basic/test.bats \ + test/functional/delete-no-version-bump/test.bats \ + test/functional/include-version-bump/test.bats \ + test/functional/update/test.bats \ + test/functional/fullfiles/test.bats \ + test/functional/pack/test.bats \ + test/functional/full-run/test.bats \ + test/functional/full-run-delta/test.bats endif if COVERAGE diff --git a/test/functional/basic.bats b/test/functional/basic/test.bats similarity index 57% rename from test/functional/basic.bats rename to test/functional/basic/test.bats index 898538c..f85b07a 100755 --- a/test/functional/basic.bats +++ b/test/functional/basic/test.bats @@ -1,23 +1,25 @@ #!/usr/bin/env bats +load "../swupdlib" + @test "create_update help output" { - run $srcdir/swupd_create_update --help + run $CREATE_UPDATE --help [ "$status" -eq 1 ] - run $srcdir/swupd_create_update -h + run $CREATE_UPDATE -h [ "$status" -eq 1 ] } @test "make_fullfiles help output" { - run $srcdir/swupd_make_fullfiles --help + run $MAKE_FULLFILES --help [ "$status" -eq 1 ] - run $srcdir/swupd_make_fullfiles -h + run $MAKE_FULLFILES -h [ "$status" -eq 1 ] } @test "make_pack help output" { - run $srcdir/swupd_make_pack --help + run $MAKE_PACK --help [ "$status" -eq 1 ] - run $srcdir/swupd_make_pack -h + run $MAKE_PACK -h [ "$status" -eq 1 ] } diff --git a/test/functional/delete-no-version-bump.bats b/test/functional/delete-no-version-bump/test.bats similarity index 71% rename from test/functional/delete-no-version-bump.bats rename to test/functional/delete-no-version-bump/test.bats index 4ebacfe..99facd8 100755 --- a/test/functional/delete-no-version-bump.bats +++ b/test/functional/delete-no-version-bump/test.bats @@ -1,11 +1,11 @@ #!/usr/bin/env bats # common functions -load swupdlib +load "../swupdlib" setup() { - DIR=$(init_web_dir "$srcdir/web-dir") - export DIR + clean_test_dir + init_test_dir init_server_ini set_latest_ver 0 @@ -32,21 +32,17 @@ setup() { } @test "delete no version bump update creation" { - sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10 + sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 10 set_latest_ver 10 - sudo $srcdir/swupd_create_update --osversion 20 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 20 + sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 20 [ 1 -eq $(grep '10 /foo' $DIR/www/10/Manifest.full | wc -l) ] [ 1 -eq $(grep '10 /foo' $DIR/www/20/Manifest.full | wc -l) ] [ 0 -eq $(grep '20 /foo' $DIR/www/20/Manifest.full | wc -l) ] } -teardown() { - sudo rm -rf $DIR -} - # vi: ft=sh ts=8 sw=2 sts=2 et tw=80 diff --git a/test/functional/full-run-delta.bats b/test/functional/full-run-delta/test.bats similarity index 79% rename from test/functional/full-run-delta.bats rename to test/functional/full-run-delta/test.bats index 0bc3ee4..c7313fa 100755 --- a/test/functional/full-run-delta.bats +++ b/test/functional/full-run-delta/test.bats @@ -1,11 +1,11 @@ #!/usr/bin/env bats # common functions -load swupdlib +load "../swupdlib" setup() { - DIR=$(init_web_dir "$srcdir/web-dir") - export DIR + clean_test_dir + init_test_dir init_server_ini set_latest_ver 0 @@ -37,22 +37,22 @@ setup() { @test "full run update creation with delta packs" { # build the first version - sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 test-bundle + sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 10 + sudo $MAKE_PACK --statedir $DIR 0 10 os-core + sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle set_latest_ver 10 # then the second version... - sudo $srcdir/swupd_create_update --osversion 20 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 20 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 test-bundle + sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 20 + sudo $MAKE_PACK --statedir $DIR 0 20 os-core + sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle # and with delta packs this time - sudo $srcdir/swupd_make_pack --statedir $DIR 10 20 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 10 20 test-bundle + sudo $MAKE_PACK --statedir $DIR 10 20 os-core + sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle # zero packs should exist (non-zero size) for both versions [ -s $DIR/www/10/pack-os-core-from-0.tar ] @@ -81,8 +81,4 @@ setup() { [[ 4 -eq $(tar -tf $DIR/www/20/pack-test-bundle-from-0.tar | wc -l) ]] } -teardown() { - sudo rm -rf $DIR -} - # vi: ft=sh ts=8 sw=2 sts=2 et tw=80 diff --git a/test/functional/full-run.bats b/test/functional/full-run/test.bats similarity index 79% rename from test/functional/full-run.bats rename to test/functional/full-run/test.bats index 762b93c..66265c8 100755 --- a/test/functional/full-run.bats +++ b/test/functional/full-run/test.bats @@ -1,11 +1,11 @@ #!/usr/bin/env bats # common functions -load swupdlib +load "../swupdlib" setup() { - DIR=$(init_web_dir "$srcdir/web-dir") - export DIR + clean_test_dir + init_test_dir init_server_ini set_latest_ver 0 @@ -20,10 +20,10 @@ setup() { } @test "full run update creation" { - sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 test-bundle + sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 10 + sudo $MAKE_PACK --statedir $DIR 0 10 os-core + sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle # zero packs should exist (non-zero size) for the build [ -s $DIR/www/10/pack-os-core-from-0.tar ] @@ -42,8 +42,4 @@ setup() { [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]] } -teardown() { - sudo rm -rf $DIR -} - # vi: ft=sh ts=8 sw=2 sts=2 et tw=80 diff --git a/test/functional/fullfiles.bats b/test/functional/fullfiles/test.bats similarity index 75% rename from test/functional/fullfiles.bats rename to test/functional/fullfiles/test.bats index 01ea6b9..87a744c 100755 --- a/test/functional/fullfiles.bats +++ b/test/functional/fullfiles/test.bats @@ -1,18 +1,20 @@ #!/usr/bin/env bats +load "../swupdlib" + @test "make_fullfiles required arg" { - run $srcdir/swupd_make_fullfiles + run $MAKE_FULLFILES [ "$status" -eq 1 ] } @test "make_fullfiles too many arguments" { # exactly one argument must be passed - run $srcdir/swupd_make_fullfiles foo bar + run $MAKE_FULLFILES foo bar [ "$status" -eq 1 ] } @test "make_fullfiles root priv check" { - run $srcdir/swupd_make_fullfiles foo + run $MAKE_FULLFILES foo [ "$status" -eq 1 ] [[ "$output" =~ "not being run as root.. exiting" ]] } diff --git a/test/functional/include-version-bump.bats b/test/functional/include-version-bump/test.bats similarity index 69% rename from test/functional/include-version-bump.bats rename to test/functional/include-version-bump/test.bats index aa2326f..b1df7e9 100755 --- a/test/functional/include-version-bump.bats +++ b/test/functional/include-version-bump/test.bats @@ -1,11 +1,11 @@ #!/usr/bin/env bats # common functions -load swupdlib +load "../swupdlib" setup() { - DIR=$(init_web_dir "$srcdir/web-dir") - export DIR + clean_test_dir + init_test_dir init_server_ini set_latest_ver 0 @@ -44,31 +44,31 @@ setup() { @test "full run update creation with delta packs" { # build the first version - sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 test-bundle + sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 10 + sudo $MAKE_PACK --statedir $DIR 0 10 os-core + sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle set_latest_ver 10 # then the second version... - sudo $srcdir/swupd_create_update --osversion 20 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 20 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 test-bundle - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 included - sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 included-two + sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 20 + sudo $MAKE_PACK --statedir $DIR 0 20 os-core + sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle + sudo $MAKE_PACK --statedir $DIR 0 20 included + sudo $MAKE_PACK --statedir $DIR 0 20 included-two set_latest_ver 20 # then the third version... - sudo $srcdir/swupd_create_update --osversion 30 --statedir $DIR --format 3 - sudo $srcdir/swupd_make_fullfiles --statedir $DIR 30 - sudo $srcdir/swupd_make_pack --statedir $DIR 0 30 os-core - sudo $srcdir/swupd_make_pack --statedir $DIR 0 30 test-bundle - sudo $srcdir/swupd_make_pack --statedir $DIR 0 30 included - sudo $srcdir/swupd_make_pack --statedir $DIR 0 30 included-two - sudo $srcdir/swupd_make_pack --statedir $DIR 0 30 included-nested + sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3 + sudo $MAKE_FULLFILES --statedir $DIR 30 + sudo $MAKE_PACK --statedir $DIR 0 30 os-core + sudo $MAKE_PACK --statedir $DIR 0 30 test-bundle + sudo $MAKE_PACK --statedir $DIR 0 30 included + sudo $MAKE_PACK --statedir $DIR 0 30 included-two + sudo $MAKE_PACK --statedir $DIR 0 30 included-nested # zero packs should exist (non-zero size) for both versions [ -s $DIR/www/10/pack-os-core-from-0.tar ] @@ -96,8 +96,4 @@ setup() { [[ 0 -eq $(ls $DIR/www/30/Manifest.included-two | wc -l) ]] } -teardown() { - sudo rm -rf $DIR -} - # vi: ft=sh ts=8 sw=2 sts=2 et tw=80 diff --git a/test/functional/pack.bats b/test/functional/pack/test.bats similarity index 66% rename from test/functional/pack.bats rename to test/functional/pack/test.bats index 70ce030..89ab699 100755 --- a/test/functional/pack.bats +++ b/test/functional/pack/test.bats @@ -1,24 +1,26 @@ #!/usr/bin/env bats +load "../swupdlib" + @test "make_pack required arg" { - run $srcdir/swupd_make_pack + run $MAKE_PACK [ "$status" -eq 1 ] } @test "make_pack too few arguments" { - run $srcdir/swupd_make_pack foo + run $MAKE_PACK foo [ "$status" -eq 1 ] - run $srcdir/swupd_make_pack foo bar + run $MAKE_PACK foo bar [ "$status" -eq 1 ] } @test "make_pack too many arguments" { - run $srcdir/swupd_make_pack foo bar foo bar + run $MAKE_PACK foo bar foo bar [ "$status" -eq 1 ] } @test "make_pack root priv check" { - run $srcdir/swupd_make_pack foo bar foo + run $MAKE_PACK foo bar foo [ "$status" -eq 1 ] [[ "$output" =~ "not being run as root.. exiting" ]] } diff --git a/test/functional/swupdlib.bash b/test/functional/swupdlib.bash index c5bd658..e072f07 100644 --- a/test/functional/swupdlib.bash +++ b/test/functional/swupdlib.bash @@ -1,14 +1,28 @@ # NOTE: source this file from a *.bats file -init_web_dir() { - # absolute path is a hard requirement right now - local dir=$(realpath $1) - mkdir -p $dir/{image,www} - echo $dir +# The location of the swupd_* binaries +export SRCDIR="$BATS_TEST_DIRNAME/../../../" + +export CREATE_UPDATE="$SRCDIR/swupd_create_update" +export MAKE_FULLFILES="$SRCDIR/swupd_make_fullfiles" +export MAKE_PACK="$SRCDIR/swupd_make_pack" + +export DIR="$BATS_TEST_DIRNAME/web-dir" + +init_test_dir() { + local testdir="$BATS_TEST_DIRNAME" + mkdir -p "$testdir"/logs + mkdir -p $DIR/{image,www} + # run swupd_* inside the directory to dump the logs + cd "$testdir"/logs +} + +clean_test_dir() { + sudo rm -rf $DIR "$BATS_TEST_DIRNAME"/logs } init_server_ini() { - cp $srcdir/server.ini $DIR + cp $SRCDIR/server.ini $DIR sed -i "s|/var/lib/update|$DIR|" $DIR/server.ini } diff --git a/test/functional/update.bats b/test/functional/update/test.bats similarity index 77% rename from test/functional/update.bats rename to test/functional/update/test.bats index e97f29c..e0fed46 100755 --- a/test/functional/update.bats +++ b/test/functional/update/test.bats @@ -1,19 +1,21 @@ #!/usr/bin/env bats +load "../swupdlib" + @test "create_update required format" { - run $srcdir/swupd_create_update + run $CREATE_UPDATE [ "$status" -eq 1 ] [[ "$output" =~ "Missing format parameter" ]] } @test "create_update required version" { - run $srcdir/swupd_create_update -F 3 + run $CREATE_UPDATE -F 3 [ "$status" -eq 1 ] [[ "$output" =~ "Missing version parameter:" ]] } @test "create_update root priv check" { - run $srcdir/swupd_create_update -F 3 -o 10 + run $CREATE_UPDATE -F 3 -o 10 [ "$status" -eq 1 ] [[ "$output" =~ "not being run as root.. exiting" ]] }