gbuild: add support for auto-creationa nd reuse of binary package.

many so-called 'external' libraries are built using the
UnpackedTarball/ExternalProject pattern, and their build is quite
stable... they can go month without any changes.
Yet some buildbot, that need to do full build, build them over and over
again.
This patch introduce the infrastructure to shortcut these build by using
a binary package of the build result

Change-Id: Ib0daf2a9d1a1f76802273c093bae7df8da4a90f8
Reviewed-on: https://gerrit.libreoffice.org/4764
Reviewed-by: Norbert Thiebaud <nthiebaud@gmail.com>
Tested-by: Norbert Thiebaud <nthiebaud@gmail.com>

3 files changed, 218 insertions, 5 deletions

diff --git a/solenv/bin/bin_library_info.sh b/solenv/bin/bin_library_info.sh
new file mode 100755
index 000000000000..29d8a4bf4dff
--- /dev/null
+++ b/solenv/bin/bin_library_info.sh
@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+#
+#    Copyright (C) 2013 Norbert Thiebaud
+#    License: GPLv3
+#
+
+do_help()
+{
+cat <<EOF
+bin_library_info.sh is a tool that create a unique filename for a binary tar file that
+contain the build of the given source tarfile. the unicity is based on the source tarfile which contains
+a md5 already and the calculated sha1 of config_host_.mk and of the tree object associated with the top_level_module
+in git.
+
+syntax: bin_library_info.sh -m|--module <top_level_module> -l|--location <TARFILE_LOCATION> -s|--srcdir <SRCDIR> -b <BUILDDIR> -r|--tarfile <LIBRARY_TARFILE> [ -m|--mode verify|name ]
+
+the default mode is 'name' which just print the assocaited binary tarfile name.
+in 'verify' mode the programe print the name if the assocaited binary tarfile exist
+and print nothing and return an error code if the file does not exist
+
+Note: --location --builddir and --srcdir are optional if they are already in the env in the form of TARFILE_LOCATION and BUILDDIR SRCDIR respectively
+EOF
+
+exit 0;
+}
+
+die()
+{
+    [ $V ] && echo "Error:" "$@"
+    exit -1;
+}
+
+
+get_config_sha()
+{
+    pushd ${SRCDIR?} > /dev/null
+    cat ${BUILDDIR?}/config_host.mk | git hash-object --stdin
+    popd ${SRCDIR?} > /dev/null
+}
+
+get_library_gbuild_sha()
+{
+    local module="$1"
+
+    pushd ${SRCDIR?} > /dev/null
+    git ls-tree HEAD | grep "\t${module?}$" | cut -f 1 | cut -d " " -f 3
+    popd ${SRCDIR?} > /dev/null
+}
+
+
+determine_binary_package_name()
+{
+    local module="$1"
+    local tarball="$2"
+    local csha=""
+    local gsha=""
+    local binfile=""
+
+    csha=$(get_config_sha)
+    gsha=$(get_library_gbuild_sha "${module?}")
+    if [ -n "${csha?}" -a -n "${gsha}" ] ; then
+        binfile="${csha?}_${gsha?}_${tarball?}.${INPATH?}.tar.gz"
+    fi
+    echo "${binfile}"
+
+}
+
+MODULE=""
+SOURCE_TARFILE=""
+MODE="name"
+V=1
+
+while [ "${1}" != "" ]; do
+    parm=${1%%=*}
+    arg=${1#*=}
+    has_arg=
+    if [ "${1}" != "${parm?}" ] ; then
+        has_arg=1
+    else
+        arg=""
+    fi
+
+    case "${parm}" in
+        -h|--help)  # display help
+            do_help
+            exit
+            ;;
+        -b|--builddir)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            BUILDDIR="${arg}"
+            ;;
+        -o|--module)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            MODULE="${arg}"
+            ;;
+
+        -l|--location)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            TARFILE_LOCATION="${arg}"
+            ;;
+        -m|--mode)
+            # test if the binary package exist
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            MODE="$arg"
+            ;;
+        -p|--platform)
+            # test if the binary package exist
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            INPATH="$arg"
+            ;;
+        -q)
+            V=0
+            ;;
+        -s|--srcdir)  # do not override the local autogen.lastrun if present
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            SRCDIR="${arg}"
+            ;;
+
+        -t|--tarfile)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            SOURCE_TARFILE="${arg}"
+            ;;
+        -*)
+            die "Invalid option $1"
+            ;;
+        *)
+            die "Invalid argument $1"
+            ;;
+    esac
+    shift
+done
+
+if [ -z "${MODULE?}" ] ; then
+    die "Missing --module"
+fi
+if [ -z "${TARFILE_LOCATION}" ] ; then
+    die "Missing --location"
+fi
+if [ -z "${SOURCE_TARFILE}" ] ; then
+    die "Missing --tarfile"
+fi
+if [ -z "${SRCDIR}" ] ; then
+    die "Missing --srcdir"
+fi
+
+
+BINARY_TARFILE="$(determine_binary_package_name ${MODULE?} ${SOURCE_TARFILE?})"
+
+if [ -z "${BINARY_TARFILE}" ] ; then
+    exit 2
+fi
+
+if [ "${MODE?}" = "verify" ] ; then
+    if [ -f "${TARFILE_LOCATION?}/${BINARY_TARFILE?}" ] ; then
+        echo "${BINARY_TARFILE?}"
+    else
+        exit 1
+    fi
+else
+    echo "${BINARY_TARFILE?}"
+fi
+
+exit 0
diff --git a/solenv/gbuild/ExternalProject.mk b/solenv/gbuild/ExternalProject.mk
index cd61cd5cf602..9b5466ba86b2 100644
--- a/solenv/gbuild/ExternalProject.mk
+++ b/solenv/gbuild/ExternalProject.mk
@@ -194,9 +194,12 @@ endef
 #
 
 define gb_ExternalProject_run
+$(if $(findstring YES,$(UNPACKED_IS_BIN_TARBALL)),\
+	touch $@,
 $(call gb_Helper_print_on_error,cd $(EXTERNAL_WORKDIR)/$(3) && \
 	$(if $(WRAPPERS),export $(WRAPPERS) &&) \
 	$(2) && touch $@,$(EXTERNAL_WORKDIR)/$(if $(3),$(3)/,)$(if $(4),$(4),$(1).log))
+)
 endef
 
 # vim: set noet sw=4 ts=4:
diff --git a/solenv/gbuild/UnpackedTarball.mk b/solenv/gbuild/UnpackedTarball.mk
index d75cc23e0c72..fdebcb97f576 100644
--- a/solenv/gbuild/UnpackedTarball.mk
+++ b/solenv/gbuild/UnpackedTarball.mk
@@ -130,7 +130,8 @@ define gb_UnpackedTarball__command
 $(call gb_Output_announce,$(2),$(true),PAT,2)
 $(call gb_Helper_abbreviate_dirs,\
 	( \
-		cd $(3) && \
+		cd $(3) \
+		$(if $(UNPACKED_IS_BIN_TARBALL),,&& \
 		$(if $(UNPACKED_PRE_ACTION),\
 			$(UNPACKED_PRE_ACTION) && \
 		) \
@@ -156,6 +157,7 @@ $(call gb_Helper_abbreviate_dirs,\
 			cp -r $(call gb_UnpackedTarball_get_dir,$(2)) $(call gb_UnpackedTarball_get_pristine_dir,$(2)) && \
 		) \
 		touch $(1) \
+		)\
 	) || \
 	( \
 		touch $(call gb_UnpackedTarball_get_preparation_target,$(2)) && \
@@ -230,14 +232,38 @@ $(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_FIX_EOL += $(addprefix $(c
 
 endef
 
+
+# Internal version of set_tarbal, mostly to avoid repeated invocation of $(shel
+define gb_UnpackedTarball_set_tarball_internal
+$(call gb_UnpackedTarget_UnpackedTarget,$(2),$(call gb_UnpackedTarball_get_dir,$(1)),$(3),$(4))
+$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarget_get_target,$(2))
+$(call gb_UnpackedTarball_get_clean_target,$(1)) : $(call gb_UnpackedTarget_get_clean_target,$(2))
+$(call gb_UnpackedTarget_get_target,$(2)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+$(if $(findstring in,$(5)),
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_IS_BIN_TARBALL := YES
+$(call gb_ExternalProject_get_state_target,$(1),%) : UNPACKED_IS_BIN_TARBALL := YES)
+$(if $(findstring out,$(5)),$(call gb_Module_get_target,$(4)) : $(gb_UnpackedTarget_TARFILE_LOCATION)/$(6)
+$(gb_UnpackedTarget_TARFILE_LOCATION)/$(6) : $(call gb_Module_get_almost_target,$(4))
+	$$(call gb_Output_announce,$(6),$(true),PKB,3)
+	if test ! -f "$$@" ; then cd $(call gb_UnpackedTarball_get_dir,) && $(GNUTAR) -czf "$$@" $(1)/ || $(GNUTAR) -czf "$$@" $(1)/ ; else touch "$$@" ; fi)
+
+endef
+
 # Set tarball name
 #
 # gb_UnpackedTarball_set_tarball unpacked tarball-name
 define gb_UnpackedTarball_set_tarball
-$(call gb_UnpackedTarget_UnpackedTarget,$(2),$(call gb_UnpackedTarball_get_dir,$(1)),$(3))
-$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarget_get_target,$(2))
-$(call gb_UnpackedTarball_get_clean_target,$(1)) : $(call gb_UnpackedTarget_get_clean_target,$(2))
-$(call gb_UnpackedTarget_get_target,$(2)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+$(if $(findstring YES,$(USE_LIBRARY_BIN_TAR)),
+$(if $(4),
+$(if $(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m verify -p "$(INPATH)"),
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m verify -p "$(INPATH)"),$(3),$(4),in),\
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),out,$(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m name -p "$(INPATH)")))
+,
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),)
+)
+,
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),)
+)
 
 endef