ci: Build ARM baremetal rootfs in native container

Doing so in an x86 container via qemu was slow, and started failing
recently after updating to a newer qemu version.

This also results in smaller arm*_test* docker images, since we need to
install fewer Debian packages in them.

As a bonus, this turns some piglit tests from fail to pass (Or maybe
they'll turn out to be flakes? They've passed at least 3 times in a
row).

Reviewed-by: Eric Anholt <eric@anholt.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9955>

6 files changed, 75 insertions, 114 deletions

diff --git a/.gitlab-ci/container/arm_build.sh b/.gitlab-ci/container/arm_build.sh
index 2798db741dd..4f6ed560514 100644
--- a/.gitlab-ci/container/arm_build.sh
+++ b/.gitlab-ci/container/arm_build.sh
@@ -49,6 +49,7 @@ apt-get -y install \
 	python3-pip \
 	python3-requests \
 	python3-setuptools \
+	u-boot-tools \
 	unzip \
 	wget \
 	xz-utils \
diff --git a/.gitlab-ci/container/arm_test-base.sh b/.gitlab-ci/container/arm_test-base.sh
index e7abafda44d..ef26837cb3e 100644
--- a/.gitlab-ci/container/arm_test-base.sh
+++ b/.gitlab-ci/container/arm_test-base.sh
@@ -3,29 +3,17 @@
 set -e
 set -o xtrace
 
-############### Install packages for building
+############### Install packages for baremetal testing
 apt-get install -y ca-certificates
 sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
-echo 'deb https://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list
 apt-get update
 
 apt-get install -y --no-remove \
         abootimg \
-        android-sdk-ext4-utils \
-        bc \
-        bison \
-        bzip2 \
-        ccache \
-        cmake \
         cpio \
-        debootstrap \
         fastboot \
-        flex \
-        g++ \
-        git \
         netcat \
         nginx-full \
-        pkg-config \
         procps \
         python3-distutils \
         python3-minimal \
@@ -33,14 +21,9 @@ apt-get install -y --no-remove \
         python3.7 \
         rsync \
         telnet \
-        u-boot-tools \
-        unzip
-
-apt install -t buster-backports -y --no-remove \
-    meson
+        unzip \
+        wget
 
 # setup nginx
 sed -i '/gzip_/ s/#\ //g' /etc/nginx/nginx.conf
 cp .gitlab-ci/bare-metal/nginx-default-site  /etc/nginx/sites-enabled/default
-
-. .gitlab-ci/container/container_post_build.sh
diff --git a/.gitlab-ci/container/baremetal_build.sh b/.gitlab-ci/container/baremetal_build.sh
index c44bf129423..aaf4c1fb2a5 100644
--- a/.gitlab-ci/container/baremetal_build.sh
+++ b/.gitlab-ci/container/baremetal_build.sh
@@ -3,60 +3,33 @@
 set -e
 set -o xtrace
 
-ROOTFS=/lava-files/rootfs-${arch}
-
-dpkg --add-architecture $arch
-apt-get update
-
-# Cross-build test deps
-BAREMETAL_EPHEMERAL=" \
-        autoconf \
-        automake \
-        crossbuild-essential-$arch \
-        git-lfs \
-        libboost-dev:$arch \
-        libdrm-dev:$arch \
-        libegl1-mesa-dev:$arch \
-        libelf-dev:$arch \
-        libexpat1-dev:$arch \
-        libffi-dev:$arch \
-        libgbm-dev:$arch \
-        libgl1-mesa-dev:$arch \
-        libgles2-mesa-dev:$arch \
-        libpciaccess-dev:$arch \
-        libpcre3-dev:$arch \
-        libpng-dev:$arch \
-        libpython3-dev:$arch \
-        libstdc++6:$arch \
-        libtinfo-dev:$arch \
-        libudev-dev:$arch \
-        libvulkan-dev:$arch \
-        libwaffle-dev:$arch \
-        libxcb-keysyms1-dev:$arch \
-        libxkbcommon-dev:$arch \
-        python3-dev \
-        qt5-default \
-        qt5-qmake \
-        qtbase5-dev:$arch \
-        "
-
-apt-get install -y --no-remove $BAREMETAL_EPHEMERAL
-
-mkdir /var/cache/apt/archives/$arch
-
-############### Create cross-files
-
-. .gitlab-ci/container/create-cross-file.sh $arch
-
-. .gitlab-ci/container/container_pre_build.sh
-
-############### Create rootfs
-KERNEL_URL=https://github.com/anholt/linux/archive/mesa-ci-2021-02-17-5.11.tar.gz
-
-DEBIAN_ARCH=$arch . .gitlab-ci/container/lava_build.sh
-
-############### Uninstall the build software
-
-apt-get purge -y $BAREMETAL_EPHEMERAL
-
-. .gitlab-ci/container/container_post_build.sh
+# Fetch the arm-built rootfs image and unpack it in our x86 container (saves
+# network transfer, disk usage, and runtime on test jobs)
+
+if wget -q --method=HEAD "${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${ARTIFACTS_SUFFIX}/done"; then
+  ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${ARTIFACTS_SUFFIX}"
+else
+  ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${CI_PROJECT_PATH}/${ARTIFACTS_SUFFIX}"
+fi
+
+wget ${ARTIFACTS_URL}/lava-rootfs.tgz -O rootfs.tgz
+mkdir -p /rootfs
+tar -C /rootfs -zxvf rootfs.tgz
+rm rootfs.tgz
+
+if [[ $arch == "arm64" ]]; then
+    mkdir -p /baremetal-files
+    pushd /baremetal-files
+
+    wget ${ARTIFACTS_URL}/Image
+    wget ${ARTIFACTS_URL}/Image.gz
+    wget ${ARTIFACTS_URL}/cheza-kernel
+
+    DEVICE_TREES="apq8016-sbc.dtb apq8096-db820c.dtb"
+
+    for DTB in $DEVICE_TREES; do
+        wget ${ARTIFACTS_URL}/$DTB
+    done
+
+    popd
+fi
diff --git a/.gitlab-ci/container/create-rootfs.sh b/.gitlab-ci/container/create-rootfs.sh
index d3dd8d46570..0297ebf4474 100644
--- a/.gitlab-ci/container/create-rootfs.sh
+++ b/.gitlab-ci/container/create-rootfs.sh
@@ -51,6 +51,7 @@ apt-get -y install --no-install-recommends \
     libxext6 \
     libxfixes3 \
     libxkbcommon0 \
+    libxrender1 \
     libxshmfence1 \
     libxxf86vm1 \
     netcat-openbsd \
diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh
index 2c6365c3d5a..ef6e0593b25 100755
--- a/.gitlab-ci/container/lava_build.sh
+++ b/.gitlab-ci/container/lava_build.sh
@@ -5,7 +5,7 @@ set -o xtrace
 
 check_minio()
 {
-    MINIO_PATH="${MINIO_HOST}/mesa-lava/$1/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}"
+    MINIO_PATH="${MINIO_HOST}/mesa-lava/$1/${MINIO_SUFFIX}/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}"
     if wget -q --method=HEAD "https://${MINIO_PATH}/done"; then
         exit
     fi
@@ -194,7 +194,7 @@ if [ -n "$INSTALL_KERNEL_MODULES" ]; then
     INSTALL_MOD_PATH=/lava-files/rootfs-${DEBIAN_ARCH}/ make modules_install
 fi
 
-if [[ ${DEBIAN_ARCH} = "arm64" ]] && which mkimage > /dev/null; then
+if [[ ${DEBIAN_ARCH} = "arm64" ]] && [[ ${MINIO_SUFFIX} = "baremetal" ]]; then
     make Image.lzma
     mkimage \
         -f auto \
@@ -204,6 +204,7 @@ if [[ ${DEBIAN_ARCH} = "arm64" ]] && which mkimage > /dev/null; then
         -C lzma\
         -b arch/arm64/boot/dts/qcom/sdm845-cheza-r3.dtb \
         /lava-files/cheza-kernel
+    KERNEL_IMAGE_NAME+=" cheza-kernel"
 fi
 
 popd
@@ -241,14 +242,10 @@ find /libdrm/ -name lib\*\.so\* | xargs cp -t /lava-files/rootfs-${DEBIAN_ARCH}/
 rm -rf /libdrm
 
 
-du -ah /lava-files/rootfs-${DEBIAN_ARCH} | sort -h | tail -100
-pushd /lava-files/rootfs-${DEBIAN_ARCH}
-  tar czf /lava-files/lava-rootfs.tgz .
-popd
-
-if [ ${DEBIAN_ARCH} = arm64 ]; then
+if [ ${DEBIAN_ARCH} = arm64 ] && [ ${MINIO_SUFFIX} = baremetal ]; then
     # Make a gzipped copy of the Image for db410c.
     gzip -k /lava-files/Image
+    KERNEL_IMAGE_NAME+=" Image.gz"
 
     # Add missing a630 firmware, added to debian packge in apr 2020
     wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_gmu.bin \
@@ -257,24 +254,26 @@ if [ ${DEBIAN_ARCH} = arm64 ]; then
          -O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_sqe.fw
 fi
 
+du -ah /lava-files/rootfs-${DEBIAN_ARCH} | sort -h | tail -100
+pushd /lava-files/rootfs-${DEBIAN_ARCH}
+  tar czf /lava-files/lava-rootfs.tgz .
+popd
+
 . .gitlab-ci/container/container_post_build.sh
 
 ############### Upload the files!
-if [ -n "$UPLOAD_FOR_LAVA" ]; then
-    ci-fairy minio login $CI_JOB_JWT
-    FILES_TO_UPLOAD="lava-rootfs.tgz \
-                     $KERNEL_IMAGE_NAME"
-
-    if [[ -n $DEVICE_TREES ]]; then
-        FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"
-    fi
+ci-fairy minio login $CI_JOB_JWT
+FILES_TO_UPLOAD="lava-rootfs.tgz \
+                 $KERNEL_IMAGE_NAME"
 
-    for f in $FILES_TO_UPLOAD; do
-        ci-fairy minio cp /lava-files/$f \
-            minio://${MINIO_PATH}/$f
-    done
-
-    touch /lava-files/done
-    ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
+if [[ -n $DEVICE_TREES ]]; then
+    FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"
 fi
 
+for f in $FILES_TO_UPLOAD; do
+    ci-fairy minio cp /lava-files/$f \
+             minio://${MINIO_PATH}/$f
+done
+
+touch /lava-files/done
+ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index 87fa587cee2..0389cc026a6 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -1,33 +1,36 @@
-variables:
-  MESA_LAVA_TAG: "2021-03-31-piglit"
-
 .kernel+rootfs:
-  stage: container-2
   extends:
     - .ci-run-policy
+  stage: container-2
   variables:
-    DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_LAVA_TAG}--${MESA_IMAGE_TAG}--${MESA_TEMPLATES_COMMIT}"
     GIT_STRATEGY: fetch
+  script:
+    - .gitlab-ci/container/lava_build.sh
+
+.kernel+rootfs-lava:
+  extends:
+    - .kernel+rootfs
+  variables:
     KERNEL_URL: "https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.10-rc2-for-mesa-ci/linux-v5.10-rc2-for-mesa-ci.tar.gz"
-    UPLOAD_FOR_LAVA: 1
+    DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${MESA_IMAGE_TAG}--${MESA_TEMPLATES_COMMIT}"
     INSTALL_KERNEL_MODULES: 1
+    MESA_ROOTFS_TAG: &lava-rootfs "2021-03-31-libxrender"
+    MINIO_SUFFIX: "lava"
     PIGLIT_BUILD_TARGETS: "piglit_replayer"
-  script:
-    - .gitlab-ci/container/lava_build.sh
 
 kernel+rootfs_amd64:
   extends:
     - .use-x86_build-base
-    - .kernel+rootfs
+    - .kernel+rootfs-lava
   image: "$FDO_BASE_IMAGE"
   variables:
     DEBIAN_ARCH: "amd64"
-    DISTRIBUTION_TAG: &distribution-tag-amd64 "${MESA_LAVA_TAG}--${MESA_BASE_TAG}--${MESA_TEMPLATES_COMMIT}"
+    DISTRIBUTION_TAG: &distribution-tag-amd64 "${MESA_ROOTFS_TAG}--${MESA_BASE_TAG}--${MESA_TEMPLATES_COMMIT}"
 
 kernel+rootfs_arm64:
   extends:
     - .use-arm_build
-    - .kernel+rootfs
+    - .kernel+rootfs-lava
   tags:
     - aarch64
   variables:
@@ -51,14 +54,15 @@ kernel+rootfs_armhf:
     FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PAGES_DOMAIN=${CI_PAGES_DOMAIN} CI_PROJECT_NAME=${CI_PROJECT_NAME} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_PROJECT_ROOT_NAMESPACE=${CI_PROJECT_ROOT_NAMESPACE} CI_JOB_JWT=${CI_JOB_JWT} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST}"
     DEQP_VERSION: gles2
     ARTIFACTS_PREFIX: "https://${MINIO_HOST}/mesa-lava/"
+    MESA_ROOTFS_TAG: *lava-rootfs
     MESA_URL: "https://${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
   script:
     # Try to use the kernel and rootfs built in mainline first, to save cycles
     - >
-      if wget -q --method=HEAD "${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}/done"; then
-        ARTIFACTS_URL="${ARTIFACTS_PREFIX}/mesa/mesa/${DISTRIBUTION_TAG}/${ARCH}"
+      if wget -q --method=HEAD "${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/lava/${DISTRIBUTION_TAG}/${ARCH}/done"; then
+        ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/lava/${DISTRIBUTION_TAG}/${ARCH}"
       else
-        ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}"
+        ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${CI_PROJECT_PATH}/lava/${DISTRIBUTION_TAG}/${ARCH}"
       fi
     - >
       artifacts/generate_lava.py \