ci: make piglit runner less noisy and show a better failure message

v2:
  - Do not silence piglit run (Michel).

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Eric Anholt <eric@anholt.net> [v1]
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8632>

2 files changed, 55 insertions, 31 deletions

diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index befd9243ef9..82489491b73 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -46,7 +46,7 @@ kernel+rootfs_armhf:
   variables:
     GIT_STRATEGY: none # testing doesn't build anything from source
     ENV_VARS: "DEQP_PARALLEL=6"
-    FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST}"
+    FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_JOB_URL=${CI_JOB_URL} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST}"
     DEQP_VERSION: gles2
     ARTIFACTS_PREFIX: "https://${MINIO_HOST}/mesa-lava/"
     MESA_URL: "https://${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh
index d59c3833c47..631f38d624e 100755
--- a/.gitlab-ci/piglit/run.sh
+++ b/.gitlab-ci/piglit/run.sh
@@ -128,6 +128,54 @@ if [ "$RUN_CMD_WRAPPER" ]; then
     RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
 fi
 
+print_red() {
+    RED='\033[0;31m'
+    NC='\033[0m' # No Color
+    printf "${RED}"
+    "$@"
+    printf "${NC}"
+}
+
+# wrapper to supress +x to avoid spamming the log
+quiet() {
+    set +x
+    "$@"
+    set -x
+}
+
+replay_minio_upload_images() {
+    find "$RESULTS/$__PREFIX" -type f -name "*.png" -printf "%P\n" \
+        | while read -r line; do
+
+        __TRACE="${line%-*-*}"
+        if grep -q "^$__PREFIX/$__TRACE: pass$" ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig"; then
+            if [ "x$CI_PROJECT_PATH" != "x$FDO_UPSTREAM_REPO" ]; then
+                continue
+            fi
+            __MINIO_PATH="$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL"
+            __DESTINATION_FILE_PATH="${line##*-}"
+            if ci-fairy minio ls "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" 2>/dev/null; then
+                continue
+            fi
+        else
+            __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
+            __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
+            # Adding to the JUnit the direct link to the diff page in
+            # the dashboard
+            __PIGLIT_TESTCASE_CLASSNAME="piglit\.trace\.$PIGLIT_REPLAY_DEVICE_NAME\.$(dirname $__TRACE | sed 's%/%\\.%g;s@%@\\%@')"
+            __PIGLIT_TESTCASE_NAME="$(basename $__TRACE | sed 's%\.%_%g;s@%@\\%@')"
+            __DASHBOARD_URL="https://tracie.freedesktop.org/dashboard/imagediff/${CI_PROJECT_PATH}/${CI_JOB_ID}/${__TRACE}"
+            sed '\%<testcase classname="'"${__PIGLIT_TESTCASE_CLASSNAME}"'" name="'"${__PIGLIT_TESTCASE_NAME}"'" status="fail"%,\%</system-out><failure type="fail"/></testcase>%{s%</system-out><failure type="fail"/></testcase>%</system-out><failure type="fail">To view the image differences visit: '"${__DASHBOARD_URL}"'</failure></testcase>%}' \
+                -i "$RESULTS"/junit.xml
+        fi
+
+        ci-fairy minio cp "$RESULTS/$__PREFIX/$line" \
+            "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
+    done
+}
+
+FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:")
+
 eval $RUN_CMD
 
 if [ $? -ne 0 ]; then
@@ -136,6 +184,7 @@ fi
 
 if [ ${PIGLIT_JUNIT_RESULTS:-0} -eq 1 ]; then
     ./piglit summary aggregate "$RESULTS" -o junit.xml
+    FAILURE_MESSAGE=$(printf "${FAILURE_MESSAGE}\n%s" "Check the JUnit report for failures at: ${CI_JOB_URL}/artifacts/file/results/junit.xml")
 fi
 
 PIGLIT_RESULTS="${PIGLIT_RESULTS:-$PIGLIT_PROFILES}"
@@ -157,34 +206,7 @@ if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
     ci-fairy minio cp "$RESULTS"/results.json.bz2 \
         "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/results.json.bz2"
 
-    find "$RESULTS/$__PREFIX" -type f -name "*.png" -printf "%P\n" \
-        | while read -r line; do
-
-        __TRACE="${line%-*-*}"
-        if grep -q "^$__PREFIX/$__TRACE: pass$" ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig"; then
-            if [ "x$CI_PROJECT_PATH" != "x$FDO_UPSTREAM_REPO" ]; then
-                continue
-            fi
-            __MINIO_PATH="$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL"
-            __DESTINATION_FILE_PATH="${line##*-}"
-            if ci-fairy minio ls "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" 2>/dev/null; then
-                continue
-            fi
-        else
-            __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
-            __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
-            # Adding to the JUnit the direct link to the diff page in
-            # the dashboard
-            __PIGLIT_TESTCASE_CLASSNAME="piglit\.trace\.$PIGLIT_REPLAY_DEVICE_NAME\.$(dirname $__TRACE | sed 's%/%\\.%g;s@%@\\%@')"
-            __PIGLIT_TESTCASE_NAME="$(basename $__TRACE | sed 's%\.%_%g;s@%@\\%@')"
-            __DASHBOARD_URL="https://tracie.freedesktop.org/dashboard/imagediff/${CI_PROJECT_PATH}/${CI_JOB_ID}/${__TRACE}"
-            sed '\%<testcase classname="'"${__PIGLIT_TESTCASE_CLASSNAME}"'" name="'"${__PIGLIT_TESTCASE_NAME}"'" status="fail"%,\%</system-out><failure type="fail"/></testcase>%{s%</system-out><failure type="fail"/></testcase>%</system-out><failure type="fail">To view the image differences visit: '"${__DASHBOARD_URL}"'</failure></testcase>%}' \
-                -i "$RESULTS"/junit.xml
-        fi
-
-        ci-fairy minio cp "$RESULTS/$__PREFIX/$line" \
-            "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
-    done
+    quiet replay_minio_upload_images
 
     ci-fairy minio cp "$RESULTS"/junit.xml \
         "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/junit.xml"
@@ -206,8 +228,10 @@ if [ ${PIGLIT_HTML_SUMMARY:-1} -eq 1 ]; then
         find "$OLDPWD"/summary -type f -name "*.html" -print0 \
             | xargs -0 sed -i 's%<img src="file://%<img src="https://'"${MINIO_HOST}${PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL}"'/%g'
     fi
+
+    FAILURE_MESSAGE=$(printf "${FAILURE_MESSAGE}\n%s" "Check the HTML summary for problems at: ${CI_JOB_URL}/artifacts/file/summary/problems.html")
 fi
 
-printf "%s\n" "Unexpected change in results:"
-diff -u ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE
+quiet print_red printf "%s\n" "$FAILURE_MESSAGE"
+quiet print_red diff -u ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE
 exit 1