From a9dd5d631729eea8686703fbb25a7a9d4c75a724 Mon Sep 17 00:00:00 2001
From: Rabin Vincent <rabin@rab.in>
Date: Fri, 18 Nov 2011 17:05:29 +0530
Subject: [PATCH 01/19] ktest: Save test output

The test output may contain useful information; save it along with the
already-saved buildlog, dmesg, and .config.

Link: http://lkml.kernel.org/r/1321616131-21352-1-git-send-email-rabin@rab.in

Signed-off-by: Rabin Vincent <rabin@rab.in>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 8b4c2535b266..e93c21cc0aa9 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -103,6 +103,7 @@ my $in_patchcheck = 0;
 my $run_test;
 my $redirect;
 my $buildlog;
+my $testlog;
 my $dmesg;
 my $monitor_fp;
 my $monitor_pid;
@@ -1021,17 +1022,19 @@ sub fail {
 	    mkpath($faildir) or
 		die "can't create $faildir";
 	}
-	if (-f "$output_config") {
-	    cp "$output_config", "$faildir/config" or
-		die "failed to copy .config";
-	}
-	if (-f $buildlog) {
-	    cp $buildlog, "$faildir/buildlog" or
-		die "failed to move $buildlog";
-	}
-	if (-f $dmesg) {
-	    cp $dmesg, "$faildir/dmesg" or
-		die "failed to move $dmesg";
+
+	my %files = (
+		"config" => $output_config,
+		"buildlog" => $buildlog,
+		"dmesg" => $dmesg,
+		"testlog" => $testlog,
+	);
+
+	while (my ($name, $source) = each(%files)) {
+		if (-f "$source") {
+			cp "$source", "$faildir/$name" or
+				die "failed to copy $source";
+		}
 	}
 
 	doprint "*** Saved info to $faildir ***\n";
@@ -1669,7 +1672,10 @@ sub child_run_test {
     $poweroff_on_error = 0;
     $die_on_failure = 1;
 
+    $redirect = "$testlog";
     run_command $run_test or $failed = 1;
+    undef $redirect;
+
     exit $failed;
 }
 
@@ -3169,6 +3175,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $target = "$ssh_user\@$machine";
 
     $buildlog = "$tmpdir/buildlog-$machine";
+    $testlog = "$tmpdir/testlog-$machine";
     $dmesg = "$tmpdir/dmesg-$machine";
     $make = "$makecmd O=$outputdir";
     $output_config = "$outputdir/.config";
@@ -3205,6 +3212,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 
     unlink $dmesg;
     unlink $buildlog;
+    unlink $testlog;
 
     if (defined($addconfig)) {
 	my $min = $minconfig;

From de5b6e3bf5e71532057fb4f1eb8ee29c5c7f11db Mon Sep 17 00:00:00 2001
From: Rabin Vincent <rabin@rab.in>
Date: Fri, 18 Nov 2011 17:05:31 +0530
Subject: [PATCH 02/19] ktest: Allow success logs to be stored

Add a STORE_SUCCESSES option, to allow success logs to be stored, for
example to double-check or otherwise post-process the test logs.

Link: http://lkml.kernel.org/r/1321616131-21352-3-git-send-email-rabin@rab.in

Signed-off-by: Rabin Vincent <rabin@rab.in>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 80 +++++++++++++++++++--------------
 tools/testing/ktest/sample.conf |  6 +++
 2 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index e93c21cc0aa9..6ef104e87ab0 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -113,6 +113,7 @@ my $bisect_sleep_time;
 my $patchcheck_sleep_time;
 my $ignore_warnings;
 my $store_failures;
+my $store_successes;
 my $test_name;
 my $timeout;
 my $booted_timeout;
@@ -976,6 +977,43 @@ sub wait_for_monitor {
     print "** Monitor flushed **\n";
 }
 
+sub save_logs {
+	my ($result, $basedir) = @_;
+	my @t = localtime;
+	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
+		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
+
+	my $type = $build_type;
+	if ($type =~ /useconfig/) {
+	    $type = "useconfig";
+	}
+
+	my $dir = "$machine-$test_type-$type-$result-$date";
+
+	$dir = "$basedir/$dir";
+
+	if (!-d $dir) {
+	    mkpath($dir) or
+		die "can't create $dir";
+	}
+
+	my %files = (
+		"config" => $output_config,
+		"buildlog" => $buildlog,
+		"dmesg" => $dmesg,
+		"testlog" => $testlog,
+	);
+
+	while (my ($name, $source) = each(%files)) {
+		if (-f "$source") {
+			cp "$source", "$dir/$name" or
+				die "failed to copy $source";
+		}
+	}
+
+	doprint "*** Saved info to $dir ***\n";
+}
+
 sub fail {
 
 	if ($die_on_failure) {
@@ -1004,40 +1042,9 @@ sub fail {
 	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 
-	return 1 if (!defined($store_failures));
-
-	my @t = localtime;
-	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
-		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
-
-	my $type = $build_type;
-	if ($type =~ /useconfig/) {
-	    $type = "useconfig";
-	}
-
-	my $dir = "$machine-$test_type-$type-fail-$date";
-	my $faildir = "$store_failures/$dir";
-
-	if (!-d $faildir) {
-	    mkpath($faildir) or
-		die "can't create $faildir";
-	}
-
-	my %files = (
-		"config" => $output_config,
-		"buildlog" => $buildlog,
-		"dmesg" => $dmesg,
-		"testlog" => $testlog,
-	);
-
-	while (my ($name, $source) = each(%files)) {
-		if (-f "$source") {
-			cp "$source", "$faildir/$name" or
-				die "failed to copy $source";
-		}
-	}
-
-	doprint "*** Saved info to $faildir ***\n";
+	if (defined($store_failures)) {
+	    save_logs "fail", $store_failures;
+        }
 
 	return 1;
 }
@@ -1643,6 +1650,10 @@ sub success {
     doprint     "*******************************************\n";
     doprint     "*******************************************\n";
 
+    if (defined($store_successes)) {
+        save_logs "success", $store_successes;
+    }
+
     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 	doprint "Reboot and wait $sleep_time seconds\n";
 	reboot $sleep_time;
@@ -3137,6 +3148,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $bisect_skip = set_test_option("BISECT_SKIP", $i);
     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
     $store_failures = set_test_option("STORE_FAILURES", $i);
+    $store_successes = set_test_option("STORE_SUCCESSES", $i);
     $test_name = set_test_option("TEST_NAME", $i);
     $timeout = set_test_option("TIMEOUT", $i);
     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index dbedfa196727..42e0eb9442e3 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -589,6 +589,12 @@
 # (default undefined)
 #STORE_FAILURES = /home/test/failures
 
+# Directory to store success directories on success. If this is not
+# set, the .config, dmesg and bootlog will not be saved if a
+# test succeeds.
+# (default undefined)
+#STORE_SUCCESSES = /home/test/successes
+
 # Build without doing a make mrproper, or removing .config
 # (default 0)
 #BUILD_NOCLEAN = 0

From 600bbf0aa2229390ba1c4ca4c5666c593ff9ef56 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Mon, 21 Nov 2011 20:12:04 -0500
Subject: [PATCH 03/19] ktest: Add default for ssh-user, build-target and
 target-image

When a user runs ktest without an argument, or the argument given
is not a config file that exists, ktest will ask the user a few
questions to create a simple ktest config file.

A few of the questions should have a default value set, that if anything
it will make it easier for the user to know what is suppose to
be in that value.

These new values are:

  SSH_USER, BUILD_TARGET and TARGET_IMAGE

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 6ef104e87ab0..05f429d68857 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -22,7 +22,6 @@ my %default;
 
 #default opts
 $default{"NUM_TESTS"}		= 1;
-$default{"REBOOT_TYPE"}		= "grub";
 $default{"TEST_TYPE"}		= "test";
 $default{"BUILD_TYPE"}		= "randconfig";
 $default{"MAKE_CMD"}		= "make";
@@ -51,7 +50,14 @@ $default{"REBOOT"}		= "ssh \$SSH_USER\@\$MACHINE reboot";
 $default{"STOP_AFTER_SUCCESS"}	= 10;
 $default{"STOP_AFTER_FAILURE"}	= 60;
 $default{"STOP_TEST_AFTER"}	= 600;
+
+# required, and we will ask users if they don't have them but we keep the default
+# value something that is common.
+$default{"REBOOT_TYPE"}		= "grub";
 $default{"LOCALVERSION"}	= "-test";
+$default{"SSH_USER"}		= "root";
+$default{"BUILD_TARGET"} 	= "arch/x86/boot/bzImage";
+$default{"TARGET_IMAGE"}	= "/boot/vmlinuz-test";
 
 my $ktest_config;
 my $version;

From 0e7a22de25212cfcaa0ba2c957e4e60eaa70fb9d Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Mon, 21 Nov 2011 20:39:33 -0500
Subject: [PATCH 04/19] ktest: When creating new config, allow the use of
 ${THIS_DIR}

Typing in a full path when you know that the path exists within
the directory your are running is tedious and unnecessary.

Allow the user to use ${PWD} if they want a dynamic path name
which will be the path that ktest.pl is executed from
or use ${THIS_DIR} which is a variable assigned `pwd` and
the the variable will exist within the config, allowing the user
to change it and affect all other paths using this variable as well

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 05f429d68857..77b464980de1 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -158,11 +158,15 @@ EOF
     ;
 $config_help{"BUILD_DIR"} = << "EOF"
  The directory that contains the Linux source code (full path).
+ You can use \${PWD} that will be the path where ktest.pl is run, or use
+ \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 EOF
     ;
 $config_help{"OUTPUT_DIR"} = << "EOF"
  The directory that the objects will be built (full path).
  (can not be same as BUILD_DIR)
+ You can use \${PWD} that will be the path where ktest.pl is run, or use
+ \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 EOF
     ;
 $config_help{"BUILD_TARGET"} = << "EOF"
@@ -282,7 +286,7 @@ sub get_ktest_config {
 		next;
 	    }
 	}
-	$entered_configs{$config} = process_variables($ans);
+	$entered_configs{$config} = ${ans};
 	last;
     }
 }
@@ -3012,6 +3016,17 @@ if (! -f $ktest_config) {
     print OUT << "EOF"
 # Generated by ktest.pl
 #
+
+# PWD is a ktest.pl variable that will result in the process working
+# directory that ktest.pl is executed in.
+
+# THIS_DIR is automatically assigned the PWD of the path that generated
+# the config file. It is best to use this variable when assigning other
+# directory paths within this directory. This allows you to easily
+# move the test cases to other locations or to other machines.
+#
+THIS_DIR := $variable{"PWD"}
+
 # Define each test with TEST_START
 # The config options below it will override the defaults
 TEST_START
@@ -3034,7 +3049,7 @@ if ($#new_configs >= 0) {
     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
     foreach my $config (@new_configs) {
 	print OUT "$config = $entered_configs{$config}\n";
-	$opt{$config} = $entered_configs{$config};
+	$opt{$config} = process_variables($entered_configs{$config});
     }
 }
 

From dad98754924735d4dfcbd49b68c00957e999c0ef Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Tue, 22 Nov 2011 20:48:57 -0500
Subject: [PATCH 05/19] ktest: Allow bisect test to restart where it left off

If a bisect is killed for some reason, have ktest detect that a bisect
is in progress and if so, allow the user to start the bisect where
it left off.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 70 ++++++++++++++++++++++++++++++++----
 1 file changed, 64 insertions(+), 6 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 77b464980de1..2ffb67c3c49d 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -239,20 +239,36 @@ $config_help{"REBOOT_SCRIPT"} = << "EOF"
 EOF
     ;
 
-sub read_yn {
-    my ($prompt) = @_;
+sub read_prompt {
+    my ($cancel, $prompt) = @_;
 
     my $ans;
 
     for (;;) {
-	print "$prompt [Y/n] ";
+	if ($cancel) {
+	    print "$prompt [y/n/C] ";
+	} else {
+	    print "$prompt [Y/n] ";
+	}
 	$ans = <STDIN>;
 	chomp $ans;
 	if ($ans =~ /^\s*$/) {
-	    $ans = "y";
+	    if ($cancel) {
+		$ans = "c";
+	    } else {
+		$ans = "y";
+	    }
 	}
 	last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
-	print "Please answer either 'y' or 'n'.\n";
+	if ($cancel) {
+	    last if ($ans =~ /^c$/i);
+	    print "Please answer either 'y', 'n' or 'c'.\n";
+	} else {
+	    print "Please answer either 'y' or 'n'.\n";
+	}
+    }
+    if ($ans =~ /^c/i) {
+	exit;
     }
     if ($ans !~ /^y$/i) {
 	return 0;
@@ -260,6 +276,18 @@ sub read_yn {
     return 1;
 }
 
+sub read_yn {
+    my ($prompt) = @_;
+
+    return read_prompt 0, $prompt;
+}
+
+sub read_ync {
+    my ($prompt) = @_;
+
+    return read_prompt 1, $prompt;
+}
+
 sub get_ktest_config {
     my ($config) = @_;
     my $ans;
@@ -1895,6 +1923,13 @@ sub run_bisect {
     }
 }
 
+sub update_bisect_replay {
+    my $tmp_log = "$tmpdir/ktest_bisect_log";
+    run_command "git bisect log > $tmp_log" or
+	die "can't create bisect log";
+    return $tmp_log;
+}
+
 sub bisect {
     my ($i) = @_;
 
@@ -1934,8 +1969,31 @@ sub bisect {
 	$type = "boot";
     }
 
+    # Check if a bisect was running
+    my $bisect_start_file = "$builddir/.git/BISECT_START";
+
     my $check = $opt{"BISECT_CHECK[$i]"};
-    if (defined($check) && $check ne "0") {
+    my $do_check = defined($check) && $check ne "0";
+
+    if ( -f $bisect_start_file ) {
+	print "Bisect in progress found\n";
+	if ($do_check) {
+	    print " If you say yes, then no checks of good or bad will be done\n";
+	}
+	if (defined($replay)) {
+	    print "** BISECT_REPLAY is defined in config file **";
+	    print " Ignore config option and perform new git bisect log?\n";
+	    if (read_ync " (yes, no, or cancel) ") {
+		$replay = update_bisect_replay;
+		$do_check = 0;
+	    }
+	} elsif (read_yn "read git log and continue?") {
+	    $replay = update_bisect_replay;
+	    $do_check = 0;
+	}
+    }
+
+    if ($do_check) {
 
 	# get current HEAD
 	my $head = get_sha1("HEAD");

From c4261d0f62ccbb42184d13b43807b36b100e8fb5 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Wed, 23 Nov 2011 13:41:18 -0500
Subject: [PATCH 06/19] ktest: Ask for type of test when creating a new config

When no argument is supplied to ktest, or the config applied does
not exist and a new config is being created, instead of just using
the default test type, give the user an option to pick the test type
of either 'build, install, or boot'. Other options may be added later
but then those would require more questions as they require more
fields. But that's for another release of ktest to add that feature.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 2ffb67c3c49d..f50b44fa8a03 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -752,6 +752,15 @@ sub __read_config {
     return $test_case;
 }
 
+sub get_test_case {
+	print "What test case would you like to run?\n";
+	print " (build, install or boot)\n";
+	print " Other tests are available but require editing the config file\n";
+	my $ans = <STDIN>;
+	chomp $ans;
+	$default{"TEST_TYPE"} = $ans;
+}
+
 sub read_config {
     my ($config) = @_;
 
@@ -766,10 +775,7 @@ sub read_config {
     # was a test specified?
     if (!$test_case) {
 	print "No test case specified.\n";
-	print "What test case would you like to run?\n";
-	my $ans = <STDIN>;
-	chomp $ans;
-	$default{"TEST_TYPE"} = $ans;
+	get_test_case;
     }
 
     # set any defaults
@@ -3070,6 +3076,7 @@ if ($#ARGV == 0) {
 }
 
 if (! -f $ktest_config) {
+    get_test_case;
     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
     print OUT << "EOF"
 # Generated by ktest.pl
@@ -3088,6 +3095,7 @@ THIS_DIR := $variable{"PWD"}
 # Define each test with TEST_START
 # The config options below it will override the defaults
 TEST_START
+TEST_TYPE = $default{"TEST_TYPE"}
 
 DEFAULTS
 EOF

From bb8474b181a715182a110c8ed2b3786ea7487f2b Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Wed, 23 Nov 2011 15:58:00 -0500
Subject: [PATCH 07/19] ktest: Do not ask for some options if the only test is
 build

When creating a ktest config or if te config only has build only
tests, some of the manditory config options are not needed.

Do not ask for them if all tests in the config file are just build
tests.

Suggested-by: Darren Hart <dvhart@linux.intel.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 43 ++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index f50b44fa8a03..7159e10ab8a4 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -22,7 +22,7 @@ my %default;
 
 #default opts
 $default{"NUM_TESTS"}		= 1;
-$default{"TEST_TYPE"}		= "test";
+$default{"TEST_TYPE"}		= "build";
 $default{"BUILD_TYPE"}		= "randconfig";
 $default{"MAKE_CMD"}		= "make";
 $default{"TIMEOUT"}		= 120;
@@ -136,6 +136,10 @@ my $localversion;
 my $iteration = 0;
 my $successes = 0;
 
+# set when a test is something other that just building
+# which would require more options.
+my $buildonly = 1;
+
 my %entered_configs;
 my %config_help;
 my %variable;
@@ -149,6 +153,7 @@ chomp ($variable{"PWD"} = `pwd`);
 
 $config_help{"MACHINE"} = << "EOF"
  The machine hostname that you will test.
+ For build only tests, it is still needed to differentiate log files.
 EOF
     ;
 $config_help{"SSH_USER"} = << "EOF"
@@ -321,15 +326,22 @@ sub get_ktest_config {
 
 sub get_ktest_configs {
     get_ktest_config("MACHINE");
-    get_ktest_config("SSH_USER");
     get_ktest_config("BUILD_DIR");
     get_ktest_config("OUTPUT_DIR");
-    get_ktest_config("BUILD_TARGET");
-    get_ktest_config("TARGET_IMAGE");
-    get_ktest_config("POWER_CYCLE");
-    get_ktest_config("CONSOLE");
+
+    # options required for other than just building a kernel
+    if (!$buildonly) {
+	get_ktest_config("SSH_USER");
+	get_ktest_config("BUILD_TARGET");
+	get_ktest_config("TARGET_IMAGE");
+	get_ktest_config("POWER_CYCLE");
+	get_ktest_config("CONSOLE");
+    }
+
     get_ktest_config("LOCALVERSION");
 
+    return if ($buildonly);
+
     my $rtype = $opt{"REBOOT_TYPE"};
 
     if (!defined($rtype)) {
@@ -388,6 +400,12 @@ sub process_variables {
 sub set_value {
     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 
+    if ($lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $rvalue ne "build") {
+	# Note if a test is something other than build, then we
+	# will need other manditory options.
+	$buildonly = 0;
+    }
+
     if (defined($opt{$lvalue})) {
 	if (!$override || defined(${$overrides}{$lvalue})) {
 	    my $extra = "";
@@ -3271,18 +3289,19 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $ENV{"SSH_USER"} = $ssh_user;
     $ENV{"MACHINE"} = $machine;
 
-    $target = "$ssh_user\@$machine";
-
     $buildlog = "$tmpdir/buildlog-$machine";
     $testlog = "$tmpdir/testlog-$machine";
     $dmesg = "$tmpdir/dmesg-$machine";
     $make = "$makecmd O=$outputdir";
     $output_config = "$outputdir/.config";
 
-    if ($reboot_type eq "grub") {
-	dodie "GRUB_MENU not defined" if (!defined($grub_menu));
-    } elsif (!defined($reboot_script)) {
-	dodie "REBOOT_SCRIPT not defined"
+    if (!$buildonly) {
+	$target = "$ssh_user\@$machine";
+	if ($reboot_type eq "grub") {
+	    dodie "GRUB_MENU not defined" if (!defined($grub_menu));
+	} elsif (!defined($reboot_script)) {
+	    dodie "REBOOT_SCRIPT not defined"
+	}
     }
 
     my $run_type = $build_type;

From dbd3783b4dd33dd7ce5c378bf5b1da27a1298735 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Wed, 23 Nov 2011 16:00:48 -0500
Subject: [PATCH 08/19] ktest: When creating a new config, ask for
 BUILD_OPTIONS

When creating a new config, ask for the BUILD_OPTIONS variable
that lets users add things like -j20 to the make.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 7159e10ab8a4..2b547d6683cd 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -140,6 +140,9 @@ my $successes = 0;
 # which would require more options.
 my $buildonly = 1;
 
+# set when creating a new config
+my $newconfig = 0;
+
 my %entered_configs;
 my %config_help;
 my %variable;
@@ -179,6 +182,11 @@ $config_help{"BUILD_TARGET"} = << "EOF"
  (relative to OUTPUT_DIR)
 EOF
     ;
+$config_help{"BUILD_OPTIONS"} = << "EOF"
+ Options to add to \"make\" when building.
+ i.e.  -j20
+EOF
+    ;
 $config_help{"TARGET_IMAGE"} = << "EOF"
  The place to put your image on the test machine.
 EOF
@@ -306,7 +314,7 @@ sub get_ktest_config {
 
     for (;;) {
 	print "$config = ";
-	if (defined($default{$config})) {
+	if (defined($default{$config}) && length($default{$config})) {
 	    print "\[$default{$config}\] ";
 	}
 	$ans = <STDIN>;
@@ -329,6 +337,10 @@ sub get_ktest_configs {
     get_ktest_config("BUILD_DIR");
     get_ktest_config("OUTPUT_DIR");
 
+    if ($newconfig) {
+	get_ktest_config("BUILD_OPTIONS");
+    }
+
     # options required for other than just building a kernel
     if (!$buildonly) {
 	get_ktest_config("SSH_USER");
@@ -3094,6 +3106,7 @@ if ($#ARGV == 0) {
 }
 
 if (! -f $ktest_config) {
+    $newconfig = 1;
     get_test_case;
     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
     print OUT << "EOF"

From 165708b273f87ac52a4564b114a6c046e6b3a02d Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Sat, 26 Nov 2011 20:56:52 -0500
Subject: [PATCH 09/19] ktest: Only ask options needed for install

If all the tests are only for build or install, do not ask
for options not needed to do the install, if the options do
not exist.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 2b547d6683cd..5d82c275e9e6 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -136,7 +136,7 @@ my $localversion;
 my $iteration = 0;
 my $successes = 0;
 
-# set when a test is something other that just building
+# set when a test is something other that just building or install
 # which would require more options.
 my $buildonly = 1;
 
@@ -343,11 +343,15 @@ sub get_ktest_configs {
 
     # options required for other than just building a kernel
     if (!$buildonly) {
+	get_ktest_config("POWER_CYCLE");
+	get_ktest_config("CONSOLE");
+    }
+
+    # options required for install and more
+    if ($buildonly != 1) {
 	get_ktest_config("SSH_USER");
 	get_ktest_config("BUILD_TARGET");
 	get_ktest_config("TARGET_IMAGE");
-	get_ktest_config("POWER_CYCLE");
-	get_ktest_config("CONSOLE");
     }
 
     get_ktest_config("LOCALVERSION");
@@ -412,10 +416,15 @@ sub process_variables {
 sub set_value {
     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 
-    if ($lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $rvalue ne "build") {
+    if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $rvalue ne "build") {
 	# Note if a test is something other than build, then we
 	# will need other manditory options.
-	$buildonly = 0;
+	if ($rvalue ne "install") {
+	    $buildonly = 0;
+	} else {
+	    # install still limits some manditory options.
+	    $buildonly = 2;
+	}
     }
 
     if (defined($opt{$lvalue})) {

From 2b29b2f8f869847d437b525d4cf027c09dc38286 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 11:25:46 -0500
Subject: [PATCH 10/19] ktest: Evaluate $KERNEL_VERSION in both install and
 post install

The install process may also need to know what the kernel version
is, to add it to the name. Evaluate it for both install and
post install.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 5d82c275e9e6..59738aa6ca73 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -1434,12 +1434,19 @@ sub monitor {
     return 1;
 }
 
+sub eval_kernel_version {
+    my ($option) = @_;
+
+    $option =~ s/\$KERNEL_VERSION/$version/g;
+
+    return $option;
+}
+
 sub do_post_install {
 
     return if (!defined($post_install));
 
-    my $cp_post_install = $post_install;
-    $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
+    my $cp_post_install = eval_kernel_version $post_install;
     run_command "$cp_post_install" or
 	dodie "Failed to run post install";
 }
@@ -1448,7 +1455,9 @@ sub install {
 
     return if ($no_install);
 
-    run_scp "$outputdir/$build_target", "$target_image" or
+    my $cp_target = eval_kernel_version $target_image;
+
+    run_scp "$outputdir/$build_target", "$cp_target" or
 	dodie "failed to copy image";
 
     my $install_mods = 0;

From cad9666980c1c1a76345f36a68e96fda3d78d857 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 11:32:52 -0500
Subject: [PATCH 11/19] ktest: Evaluate options before processing them

All options can take variables "${var}". Before doing any processing
or decision making on the content of an option, evaluate it incase
there are variables that may change the outcome.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 59738aa6ca73..04a7bb573daa 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -416,10 +416,12 @@ sub process_variables {
 sub set_value {
     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 
-    if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $rvalue ne "build") {
+    my $prvalue = process_variables($rvalue);
+
+    if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 	# Note if a test is something other than build, then we
 	# will need other manditory options.
-	if ($rvalue ne "install") {
+	if ($prvalue ne "install") {
 	    $buildonly = 0;
 	} else {
 	    # install still limits some manditory options.
@@ -435,13 +437,12 @@ sub set_value {
 	    }
 	    die "$name: $.: Option $lvalue defined more than once!\n$extra";
 	}
-	${$overrides}{$lvalue} = $rvalue;
+	${$overrides}{$lvalue} = $prvalue;
     }
     if ($rvalue =~ /^\s*$/) {
 	delete $opt{$lvalue};
     } else {
-	$rvalue = process_variables($rvalue);
-	$opt{$lvalue} = $rvalue;
+	$opt{$lvalue} = $prvalue;
     }
 }
 

From c5dacb88f0a6410b3270f77e3d1e1b159afc4adc Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 12:43:57 -0500
Subject: [PATCH 12/19] ktest: Allow overriding bisect test results

When running the ktest git bisect test, if the BISECT_TYPE is "test",
the bisect is determined to be good or bad based off of the error
code of the test that is run. Currently, if the test returns 0,
it is considered a pass (good), a non-zero is considered a fail (bad).

But it has been requested to add more options, and also change
the meanings of the error codes of the test. For example, one may
want the test to detect if the commit is not good or bad,
(maybe the bisect came to a point where the code in question
does not exist). The test could report an error code that should tell
ktest to skip the commit.

Also, a test could detect that something is horribly wrong and the
biscet should just be aborted.

The new options:

 BISECT_RET_GOOD
 BISECT_RET_BAD
 BISECT_RET_SKIP
 BISECT_RET_ABORT
 BISECT_RET_DEFAULT

have been added. The first 4 take an integer value that will
represent if the test should be considered a pass, fail, neither
good nor bad, or abort respectively.

The BISECT_RET_DEFAULT will bo whatever is not defined by the
above codes. If only BISECT_RET_DEFAULT is defined, then all tests
will do the default.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 47 +++++++++++++++++++++++++++++++++
 tools/testing/ktest/sample.conf | 36 +++++++++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 04a7bb573daa..47c28146dfc2 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -105,6 +105,11 @@ my $reverse_bisect;
 my $bisect_manual;
 my $bisect_skip;
 my $config_bisect_good;
+my $bisect_ret_good;
+my $bisect_ret_bad;
+my $bisect_ret_skip;
+my $bisect_ret_abort;
+my $bisect_ret_default;
 my $in_patchcheck = 0;
 my $run_test;
 my $redirect;
@@ -1854,6 +1859,43 @@ sub do_run_test {
     waitpid $child_pid, 0;
     $child_exit = $?;
 
+    if (!$bug && $in_bisect) {
+	if (defined($bisect_ret_good)) {
+	    if ($child_exit == $bisect_ret_good) {
+		return 1;
+	    }
+	}
+	if (defined($bisect_ret_skip)) {
+	    if ($child_exit == $bisect_ret_skip) {
+		return -1;
+	    }
+	}
+	if (defined($bisect_ret_abort)) {
+	    if ($child_exit == $bisect_ret_abort) {
+		fail "test abort" and return -2;
+	    }
+	}
+	if (defined($bisect_ret_bad)) {
+	    if ($child_exit == $bisect_ret_skip) {
+		return 0;
+	    }
+	}
+	if (defined($bisect_ret_default)) {
+	    if ($bisect_ret_default eq "good") {
+		return 1;
+	    } elsif ($bisect_ret_default eq "bad") {
+		return 0;
+	    } elsif ($bisect_ret_default eq "skip") {
+		return -1;
+	    } elsif ($bisect_ret_default eq "abort") {
+		return -2;
+	    } else {
+		fail "unknown default action: $bisect_ret_default"
+		    and return -2;
+	    }
+	}
+    }
+
     if ($bug || $child_exit) {
 	return 0 if $in_bisect;
 	fail "test failed" and return 0;
@@ -3284,6 +3326,11 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
     $bisect_skip = set_test_option("BISECT_SKIP", $i);
     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
+    $bisect_ret_good = set_test_option("BISECT_RET_GOOD", $i);
+    $bisect_ret_bad = set_test_option("BISECT_RET_BAD", $i);
+    $bisect_ret_skip = set_test_option("BISECT_RET_SKIP", $i);
+    $bisect_ret_abort = set_test_option("BISECT_RET_ABORT", $i);
+    $bisect_ret_default = set_test_option("BISECT_RET_DEFAULT", $i);
     $store_failures = set_test_option("STORE_FAILURES", $i);
     $store_successes = set_test_option("STORE_SUCCESSES", $i);
     $test_name = set_test_option("TEST_NAME", $i);
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index 42e0eb9442e3..2ff0f8c483e9 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -868,6 +868,42 @@
 #   BISECT_BAD with BISECT_CHECK = good or
 #   BISECT_CHECK = bad, respectively.
 #
+# BISECT_RET_GOOD = 0 (optional, default undefined)
+#
+#   In case the specificed test returns something other than just
+#   0 for good, and non-zero for bad, you can override 0 being
+#   good by defining BISECT_RET_GOOD.
+#
+# BISECT_RET_BAD = 1 (optional, default undefined)
+#
+#   In case the specificed test returns something other than just
+#   0 for good, and non-zero for bad, you can override non-zero being
+#   bad by defining BISECT_RET_BAD.
+#
+# BISECT_RET_ABORT = 255 (optional, default undefined)
+#
+#   If you need to abort the bisect if the test discovers something
+#   that was wrong, you can define BISECT_RET_ABORT to be the error
+#   code returned by the test in order to abort the bisect.
+#
+# BISECT_RET_SKIP = 2 (optional, default undefined)
+#
+#   If the test detects that the current commit is neither good
+#   nor bad, but something else happened (another bug detected)
+#   you can specify BISECT_RET_SKIP to an error code that the
+#   test returns when it should skip the current commit.
+#
+# BISECT_RET_DEFAULT = good (optional, default undefined)
+#
+#   You can override the default of what to do when the above
+#   options are not hit. This may be one of, "good", "bad",
+#   "abort" or "skip" (without the quotes).
+#
+#   Note, if you do not define any of the previous BISECT_RET_*
+#   and define BISECT_RET_DEFAULT, all bisects results will do
+#   what the BISECT_RET_DEFAULT has.
+#
+#
 # Example:
 #   TEST_START
 #   TEST_TYPE = bisect

From bc7c580377195f9f4da31b63fbbf52293cb4c861 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 16:29:10 -0500
Subject: [PATCH 13/19] ktest: Add options SWITCH_TO_GOOD and SWITCH_TO_TEST

For machines that do no use grub, it may be needed to update an
external image (tftp) before doing a reboot into either the
test image or the known good image.

The option SWITCH_TO_GOOD is added, where if it is defined, the
command that is specified as its value will be executed before
doing a reboot into a known good image.

The option SWITCH_TO_TEST is added, where if it is defined, the
command that is specified as its value will be executed before
doing a reboot into the test image.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 33 ++++++++++++++++++++++++++-------
 tools/testing/ktest/sample.conf | 21 +++++++++++++++++++++
 2 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 47c28146dfc2..ff21e921be25 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -79,6 +79,8 @@ my $reboot_script;
 my $power_cycle;
 my $reboot;
 my $reboot_on_error;
+my $switch_to_good;
+my $switch_to_test;
 my $poweroff_on_error;
 my $die_on_failure;
 my $powercycle_after_reboot;
@@ -964,6 +966,17 @@ sub reboot {
     }
 }
 
+sub reboot_to_good {
+    my ($time) = @_;
+
+    if (defined($switch_to_good)) {
+	run_command $switch_to_good;
+	return;
+    }
+
+    reboot $time;
+}
+
 sub do_not_reboot {
     my $i = $iteration;
 
@@ -980,7 +993,7 @@ sub dodie {
     if ($reboot_on_error && !do_not_reboot) {
 
 	doprint "REBOOTING\n";
-	reboot;
+	reboot_to_good;
 
     } elsif ($poweroff_on_error && defined($power_off)) {
 	doprint "POWERING OFF\n";
@@ -1116,7 +1129,7 @@ sub fail {
 	# no need to reboot for just building.
 	if (!do_not_reboot) {
 	    doprint "REBOOTING\n";
-	    reboot $sleep_time;
+	    reboot_to_good $sleep_time;
 	}
 
 	my $name = "";
@@ -1269,6 +1282,10 @@ sub wait_for_input
 }
 
 sub reboot_to {
+    if (defined($switch_to_test)) {
+	run_command $switch_to_test;
+    }
+
     if ($reboot_type eq "grub") {
 	run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
 	reboot;
@@ -1754,7 +1771,7 @@ sub success {
 
     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 	doprint "Reboot and wait $sleep_time seconds\n";
-	reboot $sleep_time;
+	reboot_to_good $sleep_time;
     }
 }
 
@@ -1935,7 +1952,7 @@ sub run_git_bisect {
 
 sub bisect_reboot {
     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
-    reboot $bisect_sleep_time;
+    reboot_to_good $bisect_sleep_time;
 }
 
 # returns 1 on success, 0 on failure, -1 on skip
@@ -2528,7 +2545,7 @@ sub config_bisect {
 
 sub patchcheck_reboot {
     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
-    reboot $patchcheck_sleep_time;
+    reboot_to_good $patchcheck_sleep_time;
 }
 
 sub patchcheck {
@@ -3145,7 +3162,7 @@ sub make_min_config {
 	}
 
 	doprint "Reboot and wait $sleep_time seconds\n";
-	reboot $sleep_time;
+	reboot_to_good $sleep_time;
     }
 
     success $i;
@@ -3314,6 +3331,8 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $no_install = set_test_option("NO_INSTALL", $i);
     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
+    $switch_to_good = set_test_option("SWITCH_TO_GOOD", $i);
+    $switch_to_test = set_test_option("SWITCH_TO_TEST", $i);
     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
     $power_off = set_test_option("POWER_OFF", $i);
@@ -3472,7 +3491,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 if ($opt{"POWEROFF_ON_SUCCESS"}) {
     halt;
 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
-    reboot;
+    reboot_to_good;
 }
 
 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index 2ff0f8c483e9..c8dc7575c2f5 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -468,6 +468,27 @@
 # The test will not modify that file.
 #REBOOT_TYPE = grub
 
+# If you are using a machine that doesn't boot with grub, and
+# perhaps gets its kernel from a remote server (tftp), then
+# you can use this option to update the target image with the
+# test image.
+#
+# You could also do the same with POST_INSTALL, but the difference
+# between that option and this option is that POST_INSTALL runs
+# after the install, where this one runs just before a reboot.
+# (default undefined)
+#SWITCH_TO_TEST = cp ${OUTPUT_DIR}/${BUILD_TARGET} ${TARGET_IMAGE}
+
+# If you are using a machine that doesn't boot with grub, and
+# perhaps gets its kernel from a remote server (tftp), then
+# you can use this option to update the target image with the
+# the known good image to reboot safely back into.
+#
+# This option holds a command that will execute before needing
+# to reboot to a good known image.
+# (default undefined)
+#SWITCH_TO_GOOD = ssh ${SSH_USER}/${MACHINE} cp good_image ${TARGET_IMAGE}
+
 # The min config that is needed to build for the machine
 # A nice way to create this is with the following:
 #

From 4f43e0dcafe89205d3c266fed196fd417a9fe3de Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 21:32:05 -0500
Subject: [PATCH 14/19] ktest: Change initialization of defaults hash to perl
 format

Initializing each default value by specifying the hash name is
ugly. This is one of the rare cases that the "perl way" is actually
much cleaner and easier to read.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 71 ++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index ff21e921be25..85c5d943c6db 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -18,46 +18,47 @@ $| = 1;
 my %opt;
 my %repeat_tests;
 my %repeats;
-my %default;
 
 #default opts
-$default{"NUM_TESTS"}		= 1;
-$default{"TEST_TYPE"}		= "build";
-$default{"BUILD_TYPE"}		= "randconfig";
-$default{"MAKE_CMD"}		= "make";
-$default{"TIMEOUT"}		= 120;
-$default{"TMP_DIR"}		= "/tmp/ktest/\${MACHINE}";
-$default{"SLEEP_TIME"}		= 60;	# sleep time between tests
-$default{"BUILD_NOCLEAN"}	= 0;
-$default{"REBOOT_ON_ERROR"}	= 0;
-$default{"POWEROFF_ON_ERROR"}	= 0;
-$default{"REBOOT_ON_SUCCESS"}	= 1;
-$default{"POWEROFF_ON_SUCCESS"}	= 0;
-$default{"BUILD_OPTIONS"}	= "";
-$default{"BISECT_SLEEP_TIME"}	= 60;   # sleep time between bisects
-$default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
-$default{"CLEAR_LOG"}		= 0;
-$default{"BISECT_MANUAL"}	= 0;
-$default{"BISECT_SKIP"}		= 1;
-$default{"SUCCESS_LINE"}	= "login:";
-$default{"DETECT_TRIPLE_FAULT"} = 1;
-$default{"NO_INSTALL"}		= 0;
-$default{"BOOTED_TIMEOUT"}	= 1;
-$default{"DIE_ON_FAILURE"}	= 1;
-$default{"SSH_EXEC"}		= "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
-$default{"SCP_TO_TARGET"}	= "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
-$default{"REBOOT"}		= "ssh \$SSH_USER\@\$MACHINE reboot";
-$default{"STOP_AFTER_SUCCESS"}	= 10;
-$default{"STOP_AFTER_FAILURE"}	= 60;
-$default{"STOP_TEST_AFTER"}	= 600;
+my %default = (
+    "NUM_TESTS"			=> 1,
+    "TEST_TYPE"			=> "build",
+    "BUILD_TYPE"		=> "randconfig",
+    "MAKE_CMD"			=> "make",
+    "TIMEOUT"			=> 120,
+    "TMP_DIR"			=> "/tmp/ktest/\${MACHINE}",
+    "SLEEP_TIME"		=> 60,	# sleep time between tests
+    "BUILD_NOCLEAN"		=> 0,
+    "REBOOT_ON_ERROR"		=> 0,
+    "POWEROFF_ON_ERROR"		=> 0,
+    "REBOOT_ON_SUCCESS"		=> 1,
+    "POWEROFF_ON_SUCCESS"	=> 0,
+    "BUILD_OPTIONS"		=> "",
+    "BISECT_SLEEP_TIME"		=> 60,   # sleep time between bisects
+    "PATCHCHECK_SLEEP_TIME"	=> 60, # sleep time between patch checks
+    "CLEAR_LOG"			=> 0,
+    "BISECT_MANUAL"		=> 0,
+    "BISECT_SKIP"		=> 1,
+    "SUCCESS_LINE"		=> "login:",
+    "DETECT_TRIPLE_FAULT"	=> 1,
+    "NO_INSTALL"		=> 0,
+    "BOOTED_TIMEOUT"		=> 1,
+    "DIE_ON_FAILURE"		=> 1,
+    "SSH_EXEC"			=> "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
+    "SCP_TO_TARGET"		=> "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
+    "REBOOT"			=> "ssh \$SSH_USER\@\$MACHINE reboot",
+    "STOP_AFTER_SUCCESS"	=> 10,
+    "STOP_AFTER_FAILURE"	=> 60,
+    "STOP_TEST_AFTER"		=> 600,
 
 # required, and we will ask users if they don't have them but we keep the default
 # value something that is common.
-$default{"REBOOT_TYPE"}		= "grub";
-$default{"LOCALVERSION"}	= "-test";
-$default{"SSH_USER"}		= "root";
-$default{"BUILD_TARGET"} 	= "arch/x86/boot/bzImage";
-$default{"TARGET_IMAGE"}	= "/boot/vmlinuz-test";
+    "REBOOT_TYPE"		=> "grub",
+    "LOCALVERSION"		=> "-test",
+    "SSH_USER"			=> "root",
+    "BUILD_TARGET"	 	=> "arch/x86/boot/bzImage",
+    "TARGET_IMAGE"		=> "/boot/vmlinuz-test",
+);
 
 my $ktest_config;
 my $version;

From b5f4aea687797bf9ab24f59643f88f82443b01fa Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 21:33:55 -0500
Subject: [PATCH 15/19] ktest: Have all values be set by defaults

Currently the patchcheck, bisect, and config_bisect variables
are only able to be set per test. You can not set a default
value for them.

By letting default values be set, it makes some config files
a bit easier, and also makes it easier to find typos in the
option names.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl | 86 +++++++++++++++++++++++++-----------
 1 file changed, 59 insertions(+), 27 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 85c5d943c6db..5ba9ad4248d7 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -103,7 +103,7 @@ my $output_minconfig;
 my $ignore_config;
 my $addconfig;
 my $in_bisect = 0;
-my $bisect_bad = "";
+my $bisect_bad_commit = "";
 my $reverse_bisect;
 my $bisect_manual;
 my $bisect_skip;
@@ -140,10 +140,27 @@ my $stop_after_failure;
 my $stop_test_after;
 my $build_target;
 my $target_image;
+my $checkout;
 my $localversion;
 my $iteration = 0;
 my $successes = 0;
 
+my $bisect_good;
+my $bisect_bad;
+my $bisect_type;
+my $bisect_start;
+my $bisect_replay;
+my $bisect_files;
+my $bisect_reverse;
+my $bisect_check;
+
+my $config_bisect;
+my $config_bisect_type;
+
+my $patchcheck_type;
+my $patchcheck_start;
+my $patchcheck_end;
+
 # set when a test is something other that just building or install
 # which would require more options.
 my $buildonly = 1;
@@ -1940,7 +1957,7 @@ sub run_git_bisect {
     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 	doprint "$1 [$2]\n";
     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
-	$bisect_bad = $1;
+	$bisect_bad_commit = $1;
 	doprint "Found bad commit... $1\n";
 	return 0;
     } else {
@@ -2050,16 +2067,16 @@ sub bisect {
 
     my $result;
 
-    die "BISECT_GOOD[$i] not defined\n"	if (!defined($opt{"BISECT_GOOD[$i]"}));
-    die "BISECT_BAD[$i] not defined\n"	if (!defined($opt{"BISECT_BAD[$i]"}));
-    die "BISECT_TYPE[$i] not defined\n"	if (!defined($opt{"BISECT_TYPE[$i]"}));
+    die "BISECT_GOOD[$i] not defined\n"	if (!defined($bisect_good));
+    die "BISECT_BAD[$i] not defined\n"	if (!defined($bisect_bad));
+    die "BISECT_TYPE[$i] not defined\n"	if (!defined($bisect_type));
 
-    my $good = $opt{"BISECT_GOOD[$i]"};
-    my $bad = $opt{"BISECT_BAD[$i]"};
-    my $type = $opt{"BISECT_TYPE[$i]"};
-    my $start = $opt{"BISECT_START[$i]"};
-    my $replay = $opt{"BISECT_REPLAY[$i]"};
-    my $start_files = $opt{"BISECT_FILES[$i]"};
+    my $good = $bisect_good;
+    my $bad = $bisect_bad;
+    my $type = $bisect_type;
+    my $start = $bisect_start;
+    my $replay = $bisect_replay;
+    my $start_files = $bisect_files;
 
     if (defined($start_files)) {
 	$start_files = " -- " . $start_files;
@@ -2071,8 +2088,7 @@ sub bisect {
     $good = get_sha1($good);
     $bad = get_sha1($bad);
 
-    if (defined($opt{"BISECT_REVERSE[$i]"}) &&
-	$opt{"BISECT_REVERSE[$i]"} == 1) {
+    if (defined($bisect_reverse) && $bisect_reverse == 1) {
 	doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
 	$reverse_bisect = 1;
     } else {
@@ -2087,7 +2103,7 @@ sub bisect {
     # Check if a bisect was running
     my $bisect_start_file = "$builddir/.git/BISECT_START";
 
-    my $check = $opt{"BISECT_CHECK[$i]"};
+    my $check = $bisect_check;
     my $do_check = defined($check) && $check ne "0";
 
     if ( -f $bisect_start_file ) {
@@ -2173,7 +2189,7 @@ sub bisect {
     run_command "git bisect reset" or
 	dodie "could not reset git bisect";
 
-    doprint "Bad commit was [$bisect_bad]\n";
+    doprint "Bad commit was [$bisect_bad_commit]\n";
 
     success $i;
 }
@@ -2329,7 +2345,7 @@ sub run_config_bisect {
     }
 
     doprint "***** RUN TEST ***\n";
-    my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
+    my $type = $config_bisect_type;
     my $ret;
     my %current_config;
 
@@ -2433,7 +2449,7 @@ sub run_config_bisect {
 sub config_bisect {
     my ($i) = @_;
 
-    my $start_config = $opt{"CONFIG_BISECT[$i]"};
+    my $start_config = $config_bisect;
 
     my $tmpconfig = "$tmpdir/use_config";
 
@@ -2553,22 +2569,22 @@ sub patchcheck {
     my ($i) = @_;
 
     die "PATCHCHECK_START[$i] not defined\n"
-	if (!defined($opt{"PATCHCHECK_START[$i]"}));
+	if (!defined($patchcheck_start));
     die "PATCHCHECK_TYPE[$i] not defined\n"
-	if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
+	if (!defined($patchcheck_type));
 
-    my $start = $opt{"PATCHCHECK_START[$i]"};
+    my $start = $patchcheck_start;
 
     my $end = "HEAD";
-    if (defined($opt{"PATCHCHECK_END[$i]"})) {
-	$end = $opt{"PATCHCHECK_END[$i]"};
+    if (defined($patchcheck_end)) {
+	$end = $patchcheck_end;
     }
 
     # Get the true sha1's since we can use things like HEAD~3
     $start = get_sha1($start);
     $end = get_sha1($end);
 
-    my $type = $opt{"PATCHCHECK_TYPE[$i]"};
+    my $type = $patchcheck_type;
 
     # Can't have a test without having a test to run
     if ($type eq "test" && !defined($run_test)) {
@@ -3366,9 +3382,26 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $build_target = set_test_option("BUILD_TARGET", $i);
     $ssh_exec = set_test_option("SSH_EXEC", $i);
     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
+    $checkout = set_test_option("CHECKOUT", $i);
     $target_image = set_test_option("TARGET_IMAGE", $i);
     $localversion = set_test_option("LOCALVERSION", $i);
 
+    $bisect_good = set_test_option("BISECT_GOOD", $i);
+    $bisect_bad = set_test_option("BISECT_BAD", $i);
+    $bisect_type = set_test_option("BISECT_TYPE", $i);
+    $bisect_start = set_test_option("BISECT_START", $i);
+    $bisect_replay = set_test_option("BISECT_REPLAY", $i);
+    $bisect_files = set_test_option("BISECT_FILES", $i);
+    $bisect_reverse = set_test_option("BISECT_REVERSE", $i);
+    $bisect_check = set_test_option("BISECT_CHECK", $i);
+
+    $config_bisect = set_test_option("CONFIG_BISECT", $i);
+    $config_bisect_type = set_test_option("CONFIG_BISECT_TYPE", $i);
+
+    $patchcheck_type = set_test_option("PATCHCHECK_TYPE", $i);
+    $patchcheck_start = set_test_option("PATCHCHECK_START", $i);
+    $patchcheck_end = set_test_option("PATCHCHECK_END", $i);
+
     $start_minconfig_defined = 1;
 
     if (!defined($start_minconfig)) {
@@ -3405,11 +3438,11 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 
     my $run_type = $build_type;
     if ($test_type eq "patchcheck") {
-	$run_type = $opt{"PATCHCHECK_TYPE[$i]"};
+	$run_type = $patchcheck_type;
     } elsif ($test_type eq "bisect") {
-	$run_type = $opt{"BISECT_TYPE[$i]"};
+	$run_type = $bisect_type;
     } elsif ($test_type eq "config_bisect") {
-	$run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
+	$run_type = $config_bisect_type;
     }
 
     if ($test_type eq "make_min_config") {
@@ -3441,7 +3474,6 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 	$minconfig = "$tmpdir/add_config";
     }
 
-    my $checkout = $opt{"CHECKOUT[$i]"};
     if (defined($checkout)) {
 	run_command "git checkout $checkout" or
 	    die "failed to checkout $checkout";

From 9cc9e091ed4c70535309dd9336a5ea5c2c5920aa Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 21:37:22 -0500
Subject: [PATCH 16/19] ktest: Detect typos in option names

It becomes quite annoying when you go to run a test and then
realize that you typed an option name wrong, and the test starts
doing the default action and not what you expected it to do.

It is even more annoying when you wake up the next day after
running the test over night when you discover this.

By testing if all options specified in a config file are
used by either ktest or were used in one of the option's values
we can see if there are any dangling options that were not used.
In such a case, show the user the options that were not used
and ask them if they want to continue or not.

The option IGNORE_UNUSED was also added to allow the user to
override this feature.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 208 ++++++++++++++++++++------------
 tools/testing/ktest/sample.conf |  11 ++
 2 files changed, 139 insertions(+), 80 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 5ba9ad4248d7..f360cbdbf00f 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -58,6 +58,9 @@ my %default = (
     "SSH_USER"			=> "root",
     "BUILD_TARGET"	 	=> "arch/x86/boot/bzImage",
     "TARGET_IMAGE"		=> "/boot/vmlinuz-test",
+
+    "LOG_FILE"			=> undef,
+    "IGNORE_UNUSED"		=> 0,
 );
 
 my $ktest_config;
@@ -176,6 +179,92 @@ my %force_config;
 # do not force reboots on config problems
 my $no_reboot = 1;
 
+my %option_map = (
+    "MACHINE"			=> \$machine,
+    "SSH_USER"			=> \$ssh_user,
+    "TMP_DIR"			=> \$tmpdir,
+    "OUTPUT_DIR"		=> \$outputdir,
+    "BUILD_DIR"			=> \$builddir,
+    "TEST_TYPE"			=> \$test_type,
+    "BUILD_TYPE"		=> \$build_type,
+    "BUILD_OPTIONS"		=> \$build_options,
+    "PRE_BUILD"			=> \$pre_build,
+    "POST_BUILD"		=> \$post_build,
+    "PRE_BUILD_DIE"		=> \$pre_build_die,
+    "POST_BUILD_DIE"		=> \$post_build_die,
+    "POWER_CYCLE"		=> \$power_cycle,
+    "REBOOT"			=> \$reboot,
+    "BUILD_NOCLEAN"		=> \$noclean,
+    "MIN_CONFIG"		=> \$minconfig,
+    "OUTPUT_MIN_CONFIG"		=> \$output_minconfig,
+    "START_MIN_CONFIG"		=> \$start_minconfig,
+    "IGNORE_CONFIG"		=> \$ignore_config,
+    "TEST"			=> \$run_test,
+    "ADD_CONFIG"		=> \$addconfig,
+    "REBOOT_TYPE"		=> \$reboot_type,
+    "GRUB_MENU"			=> \$grub_menu,
+    "POST_INSTALL"		=> \$post_install,
+    "NO_INSTALL"		=> \$no_install,
+    "REBOOT_SCRIPT"		=> \$reboot_script,
+    "REBOOT_ON_ERROR"		=> \$reboot_on_error,
+    "SWITCH_TO_GOOD"		=> \$switch_to_good,
+    "SWITCH_TO_TEST"		=> \$switch_to_test,
+    "POWEROFF_ON_ERROR"		=> \$poweroff_on_error,
+    "DIE_ON_FAILURE"		=> \$die_on_failure,
+    "POWER_OFF"			=> \$power_off,
+    "POWERCYCLE_AFTER_REBOOT"	=> \$powercycle_after_reboot,
+    "POWEROFF_AFTER_HALT"	=> \$poweroff_after_halt,
+    "SLEEP_TIME"		=> \$sleep_time,
+    "BISECT_SLEEP_TIME"		=> \$bisect_sleep_time,
+    "PATCHCHECK_SLEEP_TIME"	=> \$patchcheck_sleep_time,
+    "IGNORE_WARNINGS"		=> \$ignore_warnings,
+    "BISECT_MANUAL"		=> \$bisect_manual,
+    "BISECT_SKIP"		=> \$bisect_skip,
+    "CONFIG_BISECT_GOOD"	=> \$config_bisect_good,
+    "BISECT_RET_GOOD"		=> \$bisect_ret_good,
+    "BISECT_RET_BAD"		=> \$bisect_ret_bad,
+    "BISECT_RET_SKIP"		=> \$bisect_ret_skip,
+    "BISECT_RET_ABORT"		=> \$bisect_ret_abort,
+    "BISECT_RET_DEFAULT"	=> \$bisect_ret_default,
+    "STORE_FAILURES"		=> \$store_failures,
+    "STORE_SUCCESSES"		=> \$store_successes,
+    "TEST_NAME"			=> \$test_name,
+    "TIMEOUT"			=> \$timeout,
+    "BOOTED_TIMEOUT"		=> \$booted_timeout,
+    "CONSOLE"			=> \$console,
+    "DETECT_TRIPLE_FAULT"	=> \$detect_triplefault,
+    "SUCCESS_LINE"		=> \$success_line,
+    "REBOOT_SUCCESS_LINE"	=> \$reboot_success_line,
+    "STOP_AFTER_SUCCESS"	=> \$stop_after_success,
+    "STOP_AFTER_FAILURE"	=> \$stop_after_failure,
+    "STOP_TEST_AFTER"		=> \$stop_test_after,
+    "BUILD_TARGET"		=> \$build_target,
+    "SSH_EXEC"			=> \$ssh_exec,
+    "SCP_TO_TARGET"		=> \$scp_to_target,
+    "CHECKOUT"			=> \$checkout,
+    "TARGET_IMAGE"		=> \$target_image,
+    "LOCALVERSION"		=> \$localversion,
+
+    "BISECT_GOOD"		=> \$bisect_good,
+    "BISECT_BAD"		=> \$bisect_bad,
+    "BISECT_TYPE"		=> \$bisect_type,
+    "BISECT_START"		=> \$bisect_start,
+    "BISECT_REPLAY"		=> \$bisect_replay,
+    "BISECT_FILES"		=> \$bisect_files,
+    "BISECT_REVERSE"		=> \$bisect_reverse,
+    "BISECT_CHECK"		=> \$bisect_check,
+
+    "CONFIG_BISECT"		=> \$config_bisect,
+    "CONFIG_BISECT_TYPE"	=> \$config_bisect_type,
+
+    "PATCHCHECK_TYPE"		=> \$patchcheck_type,
+    "PATCHCHECK_START"		=> \$patchcheck_start,
+    "PATCHCHECK_END"		=> \$patchcheck_end,
+);
+
+# Options may be used by other options, record them.
+my %used_options;
+
 # default variables that can be used
 chomp ($variable{"PWD"} = `pwd`);
 
@@ -427,6 +516,10 @@ sub process_variables {
 	} else {
 	    # put back the origin piece.
 	    $retval = "$retval\$\{$var\}";
+	    # This could be an option that is used later, save
+	    # it so we don't warn if this option is not one of
+	    # ktests options.
+	    $used_options{$var} = 1;
 	}
 	$value = $end;
     }
@@ -850,6 +943,37 @@ sub read_config {
 	    $opt{$default} = $default{$default};
 	}
     }
+
+    if ($opt{"IGNORE_UNUSED"} == 1) {
+	return;
+    }
+
+    my %not_used;
+
+    # check if there are any stragglers (typos?)
+    foreach my $option (keys %opt) {
+	my $op = $option;
+	# remove per test labels.
+	$op =~ s/\[.*\]//;
+	if (!exists($option_map{$op}) &&
+	    !exists($default{$op}) &&
+	    !exists($used_options{$op})) {
+	    $not_used{$op} = 1;
+	}
+    }
+
+    if (%not_used) {
+	my $s = "s are";
+	$s = " is" if (keys %not_used == 1);
+	print "The following option$s not used; could be a typo:\n";
+	foreach my $option (keys %not_used) {
+	    print "$option\n";
+	}
+	print "Set IGRNORE_UNUSED = 1 to have ktest ignore unused variables\n";
+	if (!read_yn "Do you want to continue?") {
+	    exit -1;
+	}
+    }
 }
 
 sub __eval_option {
@@ -3321,86 +3445,10 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 
     my $makecmd = set_test_option("MAKE_CMD", $i);
 
-    $machine = set_test_option("MACHINE", $i);
-    $ssh_user = set_test_option("SSH_USER", $i);
-    $tmpdir = set_test_option("TMP_DIR", $i);
-    $outputdir = set_test_option("OUTPUT_DIR", $i);
-    $builddir = set_test_option("BUILD_DIR", $i);
-    $test_type = set_test_option("TEST_TYPE", $i);
-    $build_type = set_test_option("BUILD_TYPE", $i);
-    $build_options = set_test_option("BUILD_OPTIONS", $i);
-    $pre_build = set_test_option("PRE_BUILD", $i);
-    $post_build = set_test_option("POST_BUILD", $i);
-    $pre_build_die = set_test_option("PRE_BUILD_DIE", $i);
-    $post_build_die = set_test_option("POST_BUILD_DIE", $i);
-    $power_cycle = set_test_option("POWER_CYCLE", $i);
-    $reboot = set_test_option("REBOOT", $i);
-    $noclean = set_test_option("BUILD_NOCLEAN", $i);
-    $minconfig = set_test_option("MIN_CONFIG", $i);
-    $output_minconfig = set_test_option("OUTPUT_MIN_CONFIG", $i);
-    $start_minconfig = set_test_option("START_MIN_CONFIG", $i);
-    $ignore_config = set_test_option("IGNORE_CONFIG", $i);
-    $run_test = set_test_option("TEST", $i);
-    $addconfig = set_test_option("ADD_CONFIG", $i);
-    $reboot_type = set_test_option("REBOOT_TYPE", $i);
-    $grub_menu = set_test_option("GRUB_MENU", $i);
-    $post_install = set_test_option("POST_INSTALL", $i);
-    $no_install = set_test_option("NO_INSTALL", $i);
-    $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
-    $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
-    $switch_to_good = set_test_option("SWITCH_TO_GOOD", $i);
-    $switch_to_test = set_test_option("SWITCH_TO_TEST", $i);
-    $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
-    $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
-    $power_off = set_test_option("POWER_OFF", $i);
-    $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
-    $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
-    $sleep_time = set_test_option("SLEEP_TIME", $i);
-    $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
-    $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
-    $ignore_warnings = set_test_option("IGNORE_WARNINGS", $i);
-    $bisect_manual = set_test_option("BISECT_MANUAL", $i);
-    $bisect_skip = set_test_option("BISECT_SKIP", $i);
-    $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
-    $bisect_ret_good = set_test_option("BISECT_RET_GOOD", $i);
-    $bisect_ret_bad = set_test_option("BISECT_RET_BAD", $i);
-    $bisect_ret_skip = set_test_option("BISECT_RET_SKIP", $i);
-    $bisect_ret_abort = set_test_option("BISECT_RET_ABORT", $i);
-    $bisect_ret_default = set_test_option("BISECT_RET_DEFAULT", $i);
-    $store_failures = set_test_option("STORE_FAILURES", $i);
-    $store_successes = set_test_option("STORE_SUCCESSES", $i);
-    $test_name = set_test_option("TEST_NAME", $i);
-    $timeout = set_test_option("TIMEOUT", $i);
-    $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
-    $console = set_test_option("CONSOLE", $i);
-    $detect_triplefault = set_test_option("DETECT_TRIPLE_FAULT", $i);
-    $success_line = set_test_option("SUCCESS_LINE", $i);
-    $reboot_success_line = set_test_option("REBOOT_SUCCESS_LINE", $i);
-    $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
-    $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
-    $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
-    $build_target = set_test_option("BUILD_TARGET", $i);
-    $ssh_exec = set_test_option("SSH_EXEC", $i);
-    $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
-    $checkout = set_test_option("CHECKOUT", $i);
-    $target_image = set_test_option("TARGET_IMAGE", $i);
-    $localversion = set_test_option("LOCALVERSION", $i);
-
-    $bisect_good = set_test_option("BISECT_GOOD", $i);
-    $bisect_bad = set_test_option("BISECT_BAD", $i);
-    $bisect_type = set_test_option("BISECT_TYPE", $i);
-    $bisect_start = set_test_option("BISECT_START", $i);
-    $bisect_replay = set_test_option("BISECT_REPLAY", $i);
-    $bisect_files = set_test_option("BISECT_FILES", $i);
-    $bisect_reverse = set_test_option("BISECT_REVERSE", $i);
-    $bisect_check = set_test_option("BISECT_CHECK", $i);
-
-    $config_bisect = set_test_option("CONFIG_BISECT", $i);
-    $config_bisect_type = set_test_option("CONFIG_BISECT_TYPE", $i);
-
-    $patchcheck_type = set_test_option("PATCHCHECK_TYPE", $i);
-    $patchcheck_start = set_test_option("PATCHCHECK_START", $i);
-    $patchcheck_end = set_test_option("PATCHCHECK_END", $i);
+    # Load all the options into their mapped variable names
+    foreach my $opt (keys %option_map) {
+	${$option_map{$opt}} = set_test_option($opt, $i);
+    }
 
     $start_minconfig_defined = 1;
 
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index c8dc7575c2f5..86ce62c3fdca 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -727,6 +727,17 @@
 # (default 1)
 #DETECT_TRIPLE_FAULT = 0
 
+# All options in the config file should be either used by ktest
+# or could be used within a value of another option. If an option
+# in the config file is not used, ktest will warn about it and ask
+# if you want to continue.
+#
+# If you don't care if there are non-used options, enable this
+# option. Be careful though, a non-used option is usually a sign
+# of an option name being typed incorrectly.
+# (default 0)
+#IGNORE_UNUSED = 1
+
 #### Per test run options ####
 # The following options are only allowed in TEST_START sections.
 # They are ignored in the DEFAULTS sections.

From c2857cb4a855aba0c3f826bcd0dfef72abb0995b Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Thu, 22 Dec 2011 22:06:59 -0500
Subject: [PATCH 17/19] ktest: Fix compare script to test if options are not
 documented

The compare script compare-ktest-sample.pl checks for options
that are defined in ktest.pl and not documented in samples.conf,
as well as samples in samples.conf that are not used in ktest.pl.

With the switch to the hash format to initialize the ktest variables
the compare script needs to be updated to handle the change.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/compare-ktest-sample.pl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/testing/ktest/compare-ktest-sample.pl b/tools/testing/ktest/compare-ktest-sample.pl
index 9a571e71683c..a373a5bfff68 100755
--- a/tools/testing/ktest/compare-ktest-sample.pl
+++ b/tools/testing/ktest/compare-ktest-sample.pl
@@ -2,7 +2,9 @@
 
 open (IN,"ktest.pl");
 while (<IN>) {
+    # hashes are now used
     if (/\$opt\{"?([A-Z].*?)(\[.*\])?"?\}/ ||
+	/^\s*"?([A-Z].*?)"?\s*=>\s*/ ||
 	/set_test_option\("(.*?)"/) {
 	$opt{$1} = 1;
     }
@@ -11,7 +13,7 @@ close IN;
 
 open (IN, "sample.conf");
 while (<IN>) {
-    if (/^\s*#?\s*(\S+)\s*=/) {
+    if (/^\s*#?\s*([A-Z]\S*)\s*=/) {
 	$samp{$1} = 1;
     }
 }

From 96f6a0df19238fceed77ddfc33bb2d6a4e1b9be6 Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Fri, 23 Dec 2011 00:24:51 -0500
Subject: [PATCH 18/19] ktest: Still do reboot even for REBOOT_TYPE = script

The REBOOT_TYPE may be either grub or script, if it is script
it is expected that a REBOOT_SCRIPT is defined.

With the SWITCH_TO_TEST which is the complement of SWITCH_TO_GOOD,
which does basically the same thing as REBOOT_SCRIPT and but for
both grub and script, the REBOOT_SCRIPT does not need to be mandatory
anymore.

Do not require the REBOOT_SCRIPT and always run the reboot code
for both grub and script.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 11 +++--------
 tools/testing/ktest/sample.conf |  5 ++++-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index f360cbdbf00f..f81c8b3a39a8 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -485,8 +485,6 @@ sub get_ktest_configs {
 
     if ($rtype eq "grub") {
 	get_ktest_config("GRUB_MENU");
-    } else {
-	get_ktest_config("REBOOT_SCRIPT");
     }
 }
 
@@ -1430,11 +1428,10 @@ sub reboot_to {
 
     if ($reboot_type eq "grub") {
 	run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
-	reboot;
-	return;
+    } elsif (defined $reboot_script) {
+	run_command "$reboot_script";
     }
-
-    run_command "$reboot_script";
+    reboot;
 }
 
 sub get_sha1 {
@@ -3479,8 +3476,6 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 	$target = "$ssh_user\@$machine";
 	if ($reboot_type eq "grub") {
 	    dodie "GRUB_MENU not defined" if (!defined($grub_menu));
-	} elsif (!defined($reboot_script)) {
-	    dodie "REBOOT_SCRIPT not defined"
 	}
     }
 
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index 86ce62c3fdca..d7148f7c7c04 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -346,7 +346,10 @@
 #GRUB_MENU = Test Kernel
 
 # A script to reboot the target into the test kernel
-# (Only mandatory if REBOOT_TYPE = script)
+# This and SWITCH_TO_TEST are about the same, except
+# SWITCH_TO_TEST is run even for REBOOT_TYPE = grub.
+# This may be left undefined.
+# (default undefined)
 #REBOOT_SCRIPT =
 
 #### Optional Config Options (all have defaults) ####

From be405f95f0a090d1dfc26a5bb4d781c00f34479c Mon Sep 17 00:00:00 2001
From: Steven Rostedt <srostedt@redhat.com>
Date: Wed, 4 Jan 2012 21:51:59 -0500
Subject: [PATCH 19/19] ktest: Add INGORE_ERRORS to ignore warnings in boot up

When testing a kernel that has warnings, ktest.pl will fail the test
when it sees the warning. If you need to test the the kernel and want
to ignore the errors that are produced, the option IGNORE_ERRORS has
been added. When IGNORE_ERRORS is set to something other than 0, it will
ignore call traces due to WARN_ON().

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
 tools/testing/ktest/ktest.pl    | 4 +++-
 tools/testing/ktest/sample.conf | 8 ++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index f81c8b3a39a8..62a134dc421a 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -104,6 +104,7 @@ my $start_minconfig;
 my $start_minconfig_defined;
 my $output_minconfig;
 my $ignore_config;
+my $ignore_errors;
 my $addconfig;
 my $in_bisect = 0;
 my $bisect_bad_commit = "";
@@ -218,6 +219,7 @@ my %option_map = (
     "BISECT_SLEEP_TIME"		=> \$bisect_sleep_time,
     "PATCHCHECK_SLEEP_TIME"	=> \$patchcheck_sleep_time,
     "IGNORE_WARNINGS"		=> \$ignore_warnings,
+    "IGNORE_ERRORS"		=> \$ignore_errors,
     "BISECT_MANUAL"		=> \$bisect_manual,
     "BISECT_SKIP"		=> \$bisect_skip,
     "CONFIG_BISECT_GOOD"	=> \$config_bisect_good,
@@ -1529,7 +1531,7 @@ sub monitor {
 	}
 
 	if ($full_line =~ /call trace:/i) {
-	    if (!$bug && !$skip_call_trace) {
+	    if (!$ignore_errors && !$bug && !$skip_call_trace) {
 		$bug = 1;
 		$failure_start = time;
 	    }
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index d7148f7c7c04..e7119024b879 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -741,6 +741,14 @@
 # (default 0)
 #IGNORE_UNUSED = 1
 
+# When testing a kernel that happens to have WARNINGs, and call
+# traces, ktest.pl will detect these and fail a boot or test run
+# due to warnings. By setting this option, ktest will ignore
+# call traces, and will not fail a test if the kernel produces
+# an oops. Use this option with care.
+# (default 0)
+#IGNORE_ERRORS = 1
+
 #### Per test run options ####
 # The following options are only allowed in TEST_START sections.
 # They are ignored in the DEFAULTS sections.