From e70e67847c0c52a0a6e4e6d2452a032911356451 Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Mon, 18 Jun 2018 15:13:17 +0200 Subject: CI: WIP: attempt to clean up the registry before leaving According to multiple sources, referenced in https://engineering.facile.it/blog/eng/continuous-deployment-from-gitlab-ci-to-k8s-using-docker-in-docker/ The garbage collector of the registry won't clean up docker images that still have blob references. We should clean up the manifests instead of simply overwriting the tag. Note: this requires to set up a personal token with api access from the maintainers in the form of (for instance): "PERSONAL_TOKEN_bentiss" Reviewed-by: Peter Hutterer Signed-off-by: Benjamin Tissoires --- .gitlab-ci.yml | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 116 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9860ad65..b99fabd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,8 +46,8 @@ variables: MESON_BUILDDIR: builddir NINJA_ARGS: '' MESON_PARAMS: '' - FEDORA_DOCKER_IMAGE: $CI_REGISTRY/libinput/$CI_PROJECT_NAME/fedora/$FEDORA_VERSION:latest - UBUNTU_DOCKER_IMAGE: $CI_REGISTRY/libinput/$CI_PROJECT_NAME/ubuntu/$UBUNTU_VERSION:latest + FEDORA_DOCKER_IMAGE: $CI_REGISTRY/libinput/$CI_PROJECT_NAME/fedora/$FEDORA_VERSION + UBUNTU_DOCKER_IMAGE: $CI_REGISTRY/libinput/$CI_PROJECT_NAME/ubuntu/$UBUNTU_VERSION # When using docker-in-docker (dind), it's wise to use the overlayfs driver # for improved performance. DOCKER_DRIVER: overlay2 @@ -151,28 +151,28 @@ fedora:28@docker-check: variables: GIT_STRATEGY: none FEDORA_VERSION: 28 - CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE + CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE:latest <<: *docker_check fedora:27@docker-check: variables: GIT_STRATEGY: none FEDORA_VERSION: 27 - CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE + CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE:latest <<: *docker_check ubuntu:17.10@docker-check: variables: GIT_STRATEGY: none UBUNTU_VERSION: "17.10" - CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE + CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE:latest <<: *docker_check ubuntu:18.04@docker-check: variables: GIT_STRATEGY: none UBUNTU_VERSION: "18.04" - CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE + CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE:latest <<: *docker_check @@ -204,10 +204,11 @@ ubuntu:18.04@docker-check: - echo "RUN dnf install -y $FEDORA_RPMS ; dnf clean all" >> Dockerfile # create the docker image - - docker build --tag $FEDORA_DOCKER_IMAGE . + - docker build --tag $FEDORA_DOCKER_IMAGE:latest --tag $FEDORA_DOCKER_IMAGE:$CI_JOB_ID . # push the docker image to the libinput registry - - docker push $FEDORA_DOCKER_IMAGE + - docker push $FEDORA_DOCKER_IMAGE:latest + - docker push $FEDORA_DOCKER_IMAGE:$CI_JOB_ID <<: *restrict_docker_creation fedora:28@docker-prep: @@ -249,10 +250,11 @@ fedora:27@docker-prep: - echo "RUN apt-get install -y $UBUNTU_DEBS" >> Dockerfile # create the docker image - - docker build --tag $UBUNTU_DOCKER_IMAGE . + - docker build --tag $UBUNTU_DOCKER_IMAGE:latest --tag $UBUNTU_DOCKER_IMAGE:$CI_JOB_ID . # push the docker image to the libinput registry - - docker push $UBUNTU_DOCKER_IMAGE + - docker push $UBUNTU_DOCKER_IMAGE:latest + - docker push $UBUNTU_DOCKER_IMAGE:$CI_JOB_ID <<: *restrict_docker_creation ubuntu:17.10@docker-prep: @@ -308,6 +310,108 @@ ubuntu:18.04@force-docker-prep: when: manual dependencies: [] +################################################################# +# # +# docker clean stage # +# run during the check stage # +# # +################################################################# + +# +# This stage will look for the docker images we currently have in +# the registry and will remove any that are not tagged as 'latest' +# +.docker-clean: &docker_clean + stage: docker_check + image: registry.freedesktop.org/libinput/libinput/jq:latest + script: + # get the full docker image name (CURRENT_DOCKER_IMAGE still has indirections) + - DOCKER_IMAGE=$(eval echo "$CURRENT_DOCKER_IMAGE") + - REPOSITORY=$(echo $DOCKER_IMAGE | cut -f2- -d/) + + # get the r/w token from the settings to access the registry + # + # each developer needs to register a secret variable that contains + # a personal token with api access in the form of: + # PERSONAL_TOKEN_$USER (for example PERSONAL_TOKEN_bentiss) + - tokenname="PERSONAL_TOKEN_$GITLAB_USER_LOGIN" + - token=$(eval echo "\$$tokenname") + + # request a token for the registry API + - REGISTRY_TOKEN=$(curl https://gitlab.freedesktop.org/jwt/auth --get + --silent --show-error + -d client_id=docker + -d offline_token=true + -d service=container_registry + -d "scope=repository:$REPOSITORY:pull,*" + --fail + --user $GITLAB_USER_LOGIN:$token + | sed -r 's/(\{"token":"|"\})//g') + + # get the digest of the latest image + - LATEST_MANIFEST=$(curl https://$CI_REGISTRY/v2/$REPOSITORY/manifests/latest --silent + -H "accept:application/vnd.docker.distribution.manifest.v2+json" + -H "authorization:Bearer $REGISTRY_TOKEN" + --head + | grep -i "Docker-Content-Digest" + | grep -oi "sha256:\w\+") + + # get the list of tags + - TAGS=$(curl https://$CI_REGISTRY/v2/$REPOSITORY/tags/list --silent + -H "accept:application/vnd.docker.distribution.manifest.v2+json" + -H "authorization:Bearer $REGISTRY_TOKEN" + | jq -r '.tags[]') + + # iterate over the tags + - for tag in $TAGS; + do + MANIFEST=$(curl https://$CI_REGISTRY/v2/$REPOSITORY/manifests/$tag --silent + -H "accept:application/vnd.docker.distribution.manifest.v2+json" + -H "authorization:Bearer $REGISTRY_TOKEN" + --head + | grep -i "Docker-Content-Digest" + | grep -oi "sha256:\w\+"); + if test x"$MANIFEST" != x"$LATEST_MANIFEST"; + then + echo removing $tag as $MANIFEST; + curl https://$CI_REGISTRY/v2/$REPOSITORY/manifests/$MANIFEST --silent + -H "accept:application/vnd.docker.distribution.manifest.v2+json" + -H "authorization:Bearer $REGISTRY_TOKEN" + --fail --show-error -X DELETE + ;fi + ;done + dependencies: [] + allow_failure: true + <<: *restrict_docker_creation + +fedora:28@docker-clean: + variables: + GIT_STRATEGY: none + FEDORA_VERSION: 28 + CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE + <<: *docker_clean + +fedora:27@docker-clean: + variables: + GIT_STRATEGY: none + FEDORA_VERSION: 27 + CURRENT_DOCKER_IMAGE: $FEDORA_DOCKER_IMAGE + <<: *docker_clean + +ubuntu:17.10@docker-clean: + variables: + GIT_STRATEGY: none + UBUNTU_VERSION: "17.10" + CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE + <<: *docker_clean + +ubuntu:18.04@docker-clean: + variables: + GIT_STRATEGY: none + UBUNTU_VERSION: "18.04" + CURRENT_DOCKER_IMAGE: $UBUNTU_DOCKER_IMAGE + <<: *docker_clean + ################################################################# # # # build stage # @@ -320,7 +424,7 @@ ubuntu:18.04@force-docker-prep: .fedora@template: &fedora_template stage: build - image: $FEDORA_DOCKER_IMAGE + image: $FEDORA_DOCKER_IMAGE:latest <<: *default_artifacts dependencies: [] @@ -427,7 +531,7 @@ fedora:28@scan-build: .ubuntu@template: &ubuntu_template stage: build - image: $UBUNTU_DOCKER_IMAGE + image: $UBUNTU_DOCKER_IMAGE:latest <<: *default_artifacts dependencies: [] -- cgit v1.2.3