test-appliance: improve arm64 support in the LTM/KCS servers

Use cross-compilation support to build arm64 kernels using an x86
build server, instead of launching an arm build server as needed,
which has the problem that if the x86 build server is already running
when needing to build an arm kernel, or vice versa, the build request
will fail.

Also, for now, automatically disable region sharing in the LTM server
since the region sharing code doesn't know how to filter out GCE zones
that don't support arm64 VM's.

Signed-off-by: Theodore Ts'o <tytso@mit.edu>

12 files changed, 94 insertions, 73 deletions

diff --git a/selftests/ltm-kcs b/selftests/ltm-kcs
index ed0a9b03..f5738721 100755
--- a/selftests/ltm-kcs
+++ b/selftests/ltm-kcs
@@ -61,36 +61,46 @@ fi
 LTM_TEST_FILE=/tmp/ltm_test
 
 cp /dev/null $LTM_TEST_FILE
+GS_BUCKET=$(gce-xfstests get-bucket)
 
-echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 btrfs/064 " \
-     "--repo https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git " \
-     "--commit v5.15.79" >> $LTM_TEST_FILE
-
-KERNEL_DEB="$DIR/selftests/kernels/kernel-amd64.deb"
+function test-ltm ()
+{
+    ARCH=$1
+    KERNEL_DEB=$2
 
-if test -f "$KERNEL_DEB" ; then
-    if test -n "$NO_ACTION" ; then
-	$NO_ACTION $GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB \
-		   kernel-amd64.deb
+    if test -f "$KERNEL_DEB" ; then
+	if test -n "$NO_ACTION" ; then
+	    $NO_ACTION $GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB \
+		       kernel-$ARCH.deb
+	fi
+	$GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB kernel-$ARCH.deb
+	k=$"$GS_BUCKET/kernel-$ARCH.deb"
+	a=
+	if test $ARCH = "arm64" ; then
+	    a="--arch $ARCH"
+	fi
+	echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 " \
+	     "btrfs/064 $a --kernel $k" >> $LTM_TEST_FILE
     fi
-    k=$($GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB kernel-amd64.deb | \
-	    awk '{print $4}')
-    echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 " \
-	 "btrfs/064 --kernel $k" >> $LTM_TEST_FILE
-fi
+}
 
-KERNEL_DEB="$DIR/selftests/kernels/kernel-arm64.deb"
+function test-kcs ()
+{
+    ARCH=$1
 
-if test -f "$KERNEL_DEB" ; then
-    if test -n "$NO_ACTION" ; then
-	$NO_ACTION $GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB \
-		   kernel-arm64.deb
+    a=
+    if test $ARCH = "arm64" ; then
+	a="--arch $ARCH"
     fi
-    k=$($GCE_XFSTESTS upload-kernel --kernel $KERNEL_DEB kernel-arm64.deb | \
-	    awk '{print $4}')
-    echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 " \
-	 "btrfs/064 --kernel $k --no-region-shard" >> $LTM_TEST_FILE
-fi
+    echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 btrfs/064 $a" \
+	 "--repo https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git " \
+	 "--commit v5.15.79" >> $LTM_TEST_FILE
+}
+
+test-kcs amd64
+test-kcs arm64
+test-ltm amd64 $DIR/selftests/kernels/kernel-amd64.deb
+test-ltm arm64 $DIR/selftests/kernels/kernel-arm64.deb
 
 if test -n "$NO_ACTION" ; then
    echo "ltm-test script:"
diff --git a/test-appliance/files/usr/local/lib/gce-build-upload-kernel b/test-appliance/files/usr/local/lib/gce-build-upload-kernel
index 3e252151..e23e6fc8 100755
--- a/test-appliance/files/usr/local/lib/gce-build-upload-kernel
+++ b/test-appliance/files/usr/local/lib/gce-build-upload-kernel
@@ -3,18 +3,29 @@ set -vx
 # This script is used by KCS server to build and upload kernels.
 # If repository REPO_ID exists, it pulls and checkouts to COMMIT.
 # Environment variables can be set:
-# REPO_DIR GS_BUCKET GS_PATH GS_CONFIG KCONFIG_OPTS
+# REPO_DIR GS_BUCKET GS_PATH GS_CONFIG KCONFIG_OPTS KBUILD_ARCH
 
 function install_kconfig () {
     if test -z "$KCONFIG_OPTS" ; then
-	gce-xfstests install-kconfig || exit 1
+	gce-xfstests install-kconfig --arch "$KBUILD_ARCH" || exit 1
     else
 	local array=($(echo $KCONFIG_OPTS | sed -e 's/,/ /g'))
 
-	gce-xfstests install-kconfig "${array[@]}" || exit 1
+	gce-xfstests install-kconfig --arch "$KBUILD_ARCH" "${array[@]}" || exit 1
     fi
 }
 
+if test -z "$KBUILD_ARCH" ; then
+    KBUILD_ARCH=$(dpkg --print-architecture)
+fi
+KBUILD_OPTS="$KBUILD_OPTS --arch $KBUILD_ARCH"
+image=bzImage
+archdir=x86_64
+if test "$ARCH" = "arm64" ; then
+	image=Image
+	archdir=arm64
+fi
+
 if command -v ccache &> /dev/null /cache/ccache ; then
     export PATH="/usr/lib/ccache:$PATH"
     export CCACHE_DIR=/cache/ccache
@@ -28,18 +39,6 @@ if test -z "$GS_PATH"; then
     GS_PATH="gs://$GS_BUCKET/bzImage"
 fi
 
-#TODO: why was this using x86 and not x86_64?
-# x86_64 works better with new changes, is that OK to use?
-BUILD_ARCH=$(uname -m)
-case "$BUILD_ARCH" in
-    x86_64)
-	BUILD_ARCH=x86_64
-	;;
-    aarch64)
-	BUILD_ARCH=arm64
-	;;
-esac
-
 if test -n "$GS_CONFIG" ; then
     if gsutil cp "$GS_CONFIG" "$REPO_DIR/.config" ; then
 	if test $(stat -c %s "$REPO_DIR/.config") -gt 16384 ; then
@@ -53,7 +52,7 @@ if test -n "$GS_CONFIG" ; then
 else
     install_kconfig
 fi
-KERNEL_PATH="$REPO_DIR/arch/$BUILD_ARCH/boot/bzImage"
+KERNEL_PATH="$REPO_DIR/arch/$archdir/boot/$image"
 DPKG_FLAGS=
 if [[ $GS_PATH == *.deb ]]; then
     DPKG_FLAGS=--dpkg
diff --git a/test-appliance/files/usr/local/lib/gce-server/kcs/bisect.go b/test-appliance/files/usr/local/lib/gce-server/kcs/bisect.go
index f62b5cbe..cb506891 100644
--- a/test-appliance/files/usr/local/lib/gce-server/kcs/bisect.go
+++ b/test-appliance/files/usr/local/lib/gce-server/kcs/bisect.go
@@ -360,14 +360,15 @@ func (bisector *GitBisector) Build() server.ResultType {
 	bisector.testHistory = append(bisector.testHistory, newTestID)
 
 	buildLog := bisector.logDir + newTestID + ".build"
-	gsConfig := bisector.testRequest.Options.KConfig;
-	kConfigOpts := bisector.testRequest.Options.KConfigOpts;
-	kbuildOpts := bisector.testRequest.Options.KbuildOpts;
+	gsConfig := bisector.testRequest.Options.KConfig
+	kConfigOpts := bisector.testRequest.Options.KConfigOpts
+	kbuildOpts := bisector.testRequest.Options.KbuildOpts
+	arch := bisector.testRequest.Options.Arch
 
 	if logging.MOCK {
-		return MockRunBuild(bisector.repo, bisector.gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, newTestID, buildLog, bisector.log)
+		return MockRunBuild(bisector.repo, bisector.gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, arch, newTestID, buildLog, bisector.log)
 	}
-	err := RunBuild(bisector.repo, bisector.gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, newTestID, buildLog)
+	err := RunBuild(bisector.repo, bisector.gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, arch, newTestID, buildLog)
 	if !check.NoError(err, bisector.log, "Failed to build and upload kernel, skip commit") {
 		return server.Error
 	}
diff --git a/test-appliance/files/usr/local/lib/gce-server/kcs/build.go b/test-appliance/files/usr/local/lib/gce-server/kcs/build.go
index e8e2b704..cd1a665a 100644
--- a/test-appliance/files/usr/local/lib/gce-server/kcs/build.go
+++ b/test-appliance/files/usr/local/lib/gce-server/kcs/build.go
@@ -66,12 +66,13 @@ func StartBuild(c server.TaskRequest, testID string, serverLog *logrus.Entry) {
 	err = repo.Checkout(c.Options.CommitID, w)
 	check.Panic(err, cmdLog, "Failed to checkout to commit")
 
-	gsConfig := c.Options.KConfig;
-	kConfigOpts := c.Options.KConfigOpts;
-	kbuildOpts := c.Options.KbuildOpts;
+	gsConfig := c.Options.KConfig
+	kConfigOpts := c.Options.KConfigOpts
+	kbuildOpts := c.Options.KbuildOpts
+	arch := c.Options.Arch
 
 	if logging.MOCK {
-		result := MockRunBuild(repo, gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, testID, buildLog, log)
+		result := MockRunBuild(repo, gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, arch, testID, buildLog, log)
 		c.Options.GsKernel = gsPath
 		c.ExtraOptions.Requester = server.KCSTest
 		c.ExtraOptions.TestResult = result
@@ -80,7 +81,7 @@ func StartBuild(c server.TaskRequest, testID string, serverLog *logrus.Entry) {
 	}
 	cmdLog.WithField("commit", c.Options.CommitID).Info("Building kernel")
 
-	err = RunBuild(repo, gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, testID, buildLog)
+	err = RunBuild(repo, gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, arch, testID, buildLog)
 	check.Panic(err, log, "Failed to build and upload kernel")
 	log.WithField("gsPath", gsPath).Info("Kernel build and upload finished")
 
diff --git a/test-appliance/files/usr/local/lib/gce-server/kcs/mock.go b/test-appliance/files/usr/local/lib/gce-server/kcs/mock.go
index 9d587977..3e2b0db0 100644
--- a/test-appliance/files/usr/local/lib/gce-server/kcs/mock.go
+++ b/test-appliance/files/usr/local/lib/gce-server/kcs/mock.go
@@ -9,7 +9,7 @@ import (
 )
 
 // MockRunBuild runs a mock build. It reads mock.txt from repo to mock the test result.
-func MockRunBuild(repo *git.Repository, gsBucket string, gsPath string, gsConfig string, kConfigOpts string, kbuildOpts string, testID string, buildLog string, log *logrus.Entry) server.ResultType {
+func MockRunBuild(repo *git.Repository, gsBucket string, gsPath string, gsConfig string, kConfigOpts string, kbuildOpts string, arch string, testID string, buildLog string, log *logrus.Entry) server.ResultType {
 	log.Info("Start building mock kernel")
 
 	lines, _ := check.ReadLines(repo.Dir() + "mock.txt")
diff --git a/test-appliance/files/usr/local/lib/gce-server/kcs/tracker.go b/test-appliance/files/usr/local/lib/gce-server/kcs/tracker.go
index 40a410c0..7a0fc164 100644
--- a/test-appliance/files/usr/local/lib/gce-server/kcs/tracker.go
+++ b/test-appliance/files/usr/local/lib/gce-server/kcs/tracker.go
@@ -63,7 +63,7 @@ func StartTracker(instance *server.Instance, finished chan bool) {
 
 // RunBuild builds the kernel and upload the kernel image.
 // It signals the server tracker to reset the timeout timer.
-func RunBuild(repo *git.Repository, gsBucket string, gsPath string, gsConfig string, kConfigOpts string, kbuildOpts string, testID string, buildLog string) error {
+func RunBuild(repo *git.Repository, gsBucket string, gsPath string, gsConfig string, kConfigOpts string, kbuildOpts string, arch string, testID string, buildLog string) error {
 	buildLock.Lock()
 	defer buildLock.Unlock()
 	newBuild <- true
@@ -73,7 +73,7 @@ func RunBuild(repo *git.Repository, gsBucket string, gsPath string, gsConfig str
 		return err
 	}
 	defer file.Close()
-	err = repo.BuildUpload(gsBucket, gsPath, gsConfig, kConfigOpts, kbuildOpts, file)
+	err = repo.BuildUpload(gsBucket, gsPath, gsConfig, arch, kConfigOpts, kbuildOpts, file)
 
 	return err
 }
diff --git a/test-appliance/files/usr/local/lib/gce-server/ltm/shard.go b/test-appliance/files/usr/local/lib/gce-server/ltm/shard.go
index 10d8a368..25af29cd 100644
--- a/test-appliance/files/usr/local/lib/gce-server/ltm/shard.go
+++ b/test-appliance/files/usr/local/lib/gce-server/ltm/shard.go
@@ -91,6 +91,9 @@ func NewShardWorker(sharder *ShardScheduler, shardID string, config string, zone
 		"--no-email",
 		"-c", config,
 	}
+	if sharder.arch != "" {
+		shard.args = append(shard.args, "--arch", sharder.arch)
+	}
 	shard.args = append(shard.args, sharder.validArgs...)
 
 	var defaultProj bool = true
diff --git a/test-appliance/files/usr/local/lib/gce-server/ltm/sharder.go b/test-appliance/files/usr/local/lib/gce-server/ltm/sharder.go
index a6cec8e1..957efa98 100644
--- a/test-appliance/files/usr/local/lib/gce-server/ltm/sharder.go
+++ b/test-appliance/files/usr/local/lib/gce-server/ltm/sharder.go
@@ -49,6 +49,7 @@ type ShardScheduler struct {
 	bucketSubdir   string
 	gsKernel       string
 	kernelVersion  string
+	arch	       string
 	reportReceiver string
 	maxShards      int
 	keepDeadVM     bool
@@ -117,6 +118,7 @@ func NewShardScheduler(c server.TaskRequest, testID string) *ShardScheduler {
 		bucketSubdir:   bucketSubdir,
 		gsKernel:       c.Options.GsKernel,
 		kernelVersion:  "unknown_kernel_version",
+		arch:		c.Options.Arch,
 		reportReceiver: c.Options.ReportEmail,
 		maxShards:      0,
 		keepDeadVM:     false,
@@ -149,6 +151,13 @@ func NewShardScheduler(c server.TaskRequest, testID string) *ShardScheduler {
 	check.Panic(err, log, "Failed to connect to GCE service")
 
 	regionShard := !c.Options.NoRegionShard
+	// This is a hack because RegionSharding doesn't know how to
+	// exclude zones that don't have arm64 machine types.  More
+	// generally, if the user has specified a specific machtype,
+	// region sharding doesn't handle that case as well either.
+	if c.Options.Arch == "arm64" {
+		regionShard = false
+	}
 	if regionShard {
 		sharder.initRegionSharding()
 	} else {
diff --git a/test-appliance/files/usr/local/lib/gce-server/util/git/git.go b/test-appliance/files/usr/local/lib/gce-server/util/git/git.go
index 6b796729..d29be98f 100644
--- a/test-appliance/files/usr/local/lib/gce-server/util/git/git.go
+++ b/test-appliance/files/usr/local/lib/gce-server/util/git/git.go
@@ -301,7 +301,7 @@ func (repo *Repository) BisectReset(writer io.Writer) error {
 
 // BuildUpload builds the current kernel code and uploads image to GS.
 // Script output is written into a given writer.
-func (repo *Repository) BuildUpload(gsBucket string, gsPath string, gsConfig string, kConfigOpts string, kbuildOpts string, writer io.Writer) error {
+func (repo *Repository) BuildUpload(gsBucket string, gsPath string, gsConfig string, arch string, kConfigOpts string, kbuildOpts string, writer io.Writer) error {
 	repo.lock.Lock()
 	defer repo.lock.Unlock()
 	if !check.DirExists(repo.dir) {
@@ -316,6 +316,7 @@ func (repo *Repository) BuildUpload(gsBucket string, gsPath string, gsConfig str
 		"GS_CONFIG": gsConfig,
 		"KCONFIG_OPTS": kConfigOpts,
 		"KBUILD_OPTS": kbuildOpts,
+		"KBUILD_ARCH": arch,
 	}
 
 	err := check.Run(cmd, repo.dir, env, writer, writer)
diff --git a/test-appliance/files/usr/local/lib/gce-server/util/parser/parser.go b/test-appliance/files/usr/local/lib/gce-server/util/parser/parser.go
index 2403ab1c..81ff5fa0 100644
--- a/test-appliance/files/usr/local/lib/gce-server/util/parser/parser.go
+++ b/test-appliance/files/usr/local/lib/gce-server/util/parser/parser.go
@@ -32,6 +32,7 @@ var invalidOpts = []string{
 	"-r",
 	"--machtype",
 	"--kernel",
+	"--arch",
 	"--commit",
 	"--config",
 	"--kconfig-opts",
diff --git a/test-appliance/files/usr/local/lib/gce-server/util/server/info.go b/test-appliance/files/usr/local/lib/gce-server/util/server/info.go
index 8777fd04..690a3b80 100644
--- a/test-appliance/files/usr/local/lib/gce-server/util/server/info.go
+++ b/test-appliance/files/usr/local/lib/gce-server/util/server/info.go
@@ -138,7 +138,7 @@ type StatusResponse struct {
 func InternalQuery(log *logrus.Entry) StatusResponse {
 	log.Info("Sending status query to KCS")
 
-	active := accessKCS("", log, false)
+	active := accessKCS(log, false)
 	if !active {
 		log.Info("KCS is not running, skipping query")
 		return StatusResponse{}
diff --git a/test-appliance/files/usr/local/lib/gce-server/util/server/server.go b/test-appliance/files/usr/local/lib/gce-server/util/server/server.go
index 3b59b8ac..4c9122e8 100644
--- a/test-appliance/files/usr/local/lib/gce-server/util/server/server.go
+++ b/test-appliance/files/usr/local/lib/gce-server/util/server/server.go
@@ -423,7 +423,7 @@ func SendInternalRequest(c TaskRequest, log *logrus.Entry, toKCS bool) {
 
 	var config *gcp.Config
 	if toKCS {
-		accessKCS(c.Options.Arch, log, true)
+		accessKCS(log, true)
 		gcp.Update()
 		config = gcp.KCSConfig
 
@@ -447,6 +447,12 @@ func SendInternalRequest(c TaskRequest, log *logrus.Entry, toKCS bool) {
 	}
 	c.ExtraOptions.Password = password
 
+	log.WithFields(logrus.Fields{
+		"cmdLine":      c.CmdLine,
+		"options":      c.Options,
+		"extraOptions": c.ExtraOptions,
+	}).Info("Sending request")
+
 	js, err := json.Marshal(c)
 	check.Panic(err, log, "Failed to encode json request body")
 	req, err := http.NewRequest("POST", url, bytes.NewBuffer(js))
@@ -518,7 +524,7 @@ if launch is true, it attempts to launch KCS if it's not running.
 KCS is assumed to be always launched by LTM instead of user.
 It checks KCS's metadata to ensure it's not in the process of shutting down.
 */
-func accessKCS(arch string, log *logrus.Entry, launch bool) bool {
+func accessKCS(log *logrus.Entry, launch bool) bool {
 	log.Info("Launching KCS server")
 
 	zone, err := gcp.GceConfig.Get("GCE_ZONE")
@@ -536,7 +542,7 @@ func accessKCS(arch string, log *logrus.Entry, launch bool) bool {
 			if gcp.NotFound(err) {
 				if launch {
 					log.Info("KCS is not running, launching it")
-					runLaunchKCS(arch, log)
+					runLaunchKCS(log)
 					return true
 				}
 				log.Info("KCS is not running")
@@ -559,7 +565,7 @@ func accessKCS(arch string, log *logrus.Entry, launch bool) bool {
 		if active {
 			log.Info("KCS is running")
 			if gcp.KCSConfig == nil {
-				runLaunchKCS(arch, log)
+				runLaunchKCS(log)
 			}
 			return true
 		} else if !launch {
@@ -572,28 +578,18 @@ func accessKCS(arch string, log *logrus.Entry, launch bool) bool {
 	return false
 }
 
-func runLaunchKCS(arch string, log *logrus.Entry) {
+func runLaunchKCS(log *logrus.Entry) {
 	launchLock.Lock()
 	defer launchLock.Unlock()
-	var cmd *exec.Cmd
 
-	if arch != "" {
-		cmd = exec.Command("gce-xfstests", "launch-kcs", "--arch", arch)
-	} else {
-		cmd = exec.Command("gce-xfstests", "launch-kcs")
-	}
+	cmd := exec.Command("gce-xfstests", "launch-kcs")
 	cmdLog := log.WithField("cmd", cmd.String())
 	w := cmdLog.Writer()
 	defer w.Close()
 	output, err := check.LimitedOutput(cmd, check.RootDir, check.EmptyEnv, w)
-	if err != nil {
-		if strings.HasPrefix(output, "KCS server is already running on") {
-			cmdLog.WithField("output", output).WithError(err).Panic(
-			"KCS server already running but with wrong architecture")
-		} else if !strings.HasPrefix(output, "The KCS instance already exists!") {
-			cmdLog.WithField("output", output).WithError(err).Panic(
+	if err != nil && !strings.HasPrefix(output, "The KCS instance already exists!") {
+		cmdLog.WithField("output", output).WithError(err).Panic(
 			"Failed to fetch LTM config file")
-		}
 	}
 }