From 8512f4ca090c85477a6670438aeefe7fdfcf8a98 Mon Sep 17 00:00:00 2001 From: Luboš Luňák Date: Wed, 19 Feb 2020 22:54:50 +0100 Subject: build nss using their new build system (gyp/ninja-based) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This requires installed ninja, gyp is included as source. This allows nss be built as a parallel build, unlike the old Makefile build system. Since gyp internally uses python, even recursively, this requires more complicated setup in case our internal python is used. Moreover gyp itself seems to be kind of deprecated itself and hasn't been ported to python3 yet, so that needs patching too. So far only easy Unix-like systems are converted, Windows I'll do later, the more complicated systems I'll leave to whoever has access to them. Change-Id: I358baad7690d2aa6df44bafa9244dc7cc828fc3f Reviewed-on: https://gerrit.libreoffice.org/c/core/+/90115 Tested-by: Jenkins Reviewed-by: Luboš Luňák --- Makefile.fetch | 1 + configure.ac | 4 + download.lst | 2 + external/nss/ExternalProject_nss.mk | 26 +++++- external/nss/Module_nss.mk | 1 + external/nss/UnpackedTarball_gyp.mk | 18 ++++ external/nss/UnpackedTarball_nss.mk | 2 + external/nss/buildsystem.patch.0 | 32 +++++++ external/nss/fix-cc-detect-with-ccache.patch.0 | 11 +++ external/nss/gyp-buildsystem.patch.0 | 119 +++++++++++++++++++++++++ external/nss/setup-python | 3 + external/python3/ExternalPackage_python3.mk | 1 + 12 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 external/nss/UnpackedTarball_gyp.mk create mode 100644 external/nss/buildsystem.patch.0 create mode 100644 external/nss/fix-cc-detect-with-ccache.patch.0 create mode 100644 external/nss/gyp-buildsystem.patch.0 create mode 100755 external/nss/setup-python diff --git a/Makefile.fetch b/Makefile.fetch index cfa3dc58c3a5..13661978de06 100644 --- a/Makefile.fetch +++ b/Makefile.fetch @@ -131,6 +131,7 @@ $(WORKDIR)/download: $(BUILDDIR)/config_$(gb_Side).mk $(SRCDIR)/download.lst $(S $(call fetch_Optional,GLM,GLM_TARBALL) \ $(call fetch_Optional,GPGMEPP,GPGME_TARBALL) \ $(call fetch_Optional,GRAPHITE,GRAPHITE_TARBALL) \ + $(call fetch_Optional,GYP,GYP_TARBALL) \ $(call fetch_Optional,HARFBUZZ,HARFBUZZ_TARBALL) \ $(call fetch_Optional,HSQLDB,HSQLDB_TARBALL) \ $(call fetch_Optional,HUNSPELL,HUNSPELL_TARBALL) \ diff --git a/configure.ac b/configure.ac index 1cf56da09c0a..34708eab7e8f 100644 --- a/configure.ac +++ b/configure.ac @@ -9667,6 +9667,10 @@ if test $_os != iOS -a "$enable_fuzzers" != "yes"; then AC_DEFINE(HAVE_FEATURE_NSS) ENABLE_NSS="TRUE" AC_DEFINE(ENABLE_NSS) + # internal nss uses gyp + if test -z "$SYSTEM_NSS"; then + BUILD_TYPE="$BUILD_TYPE GYP" + fi elif test $_os != iOS ; then with_tls=openssl fi diff --git a/download.lst b/download.lst index 9983fc0386b1..1734dbd3ae27 100644 --- a/download.lst +++ b/download.lst @@ -99,6 +99,8 @@ export GPGME_SHA256SUM := 1b29fedb8bfad775e70eafac5b0590621683b2d9869db994568e64 export GPGME_TARBALL := gpgme-1.9.0.tar.bz2 export GRAPHITE_SHA256SUM := d47d387161db7f7ebade1920aa7cbdc797e79772597d8b55e80b58d1071bcc36 export GRAPHITE_TARBALL := graphite2-minimal-1.3.13.tgz +export GYP_SHA256SUM := 53cb1f2b19048544950422cd86c6093ba2c4ff9748e27cc560dfff9791700c9d +export GYP_TARBALL := gyp-e87d37d6bce611abed35e854d5ae1a401e9ce04c.tar.xz export HARFBUZZ_SHA256SUM := 9cf7d117548265f95ca884e2f4c9fafaf4e17d45a67b11107147b79eed76c966 export HARFBUZZ_TARBALL := harfbuzz-2.6.0.tar.xz export HSQLDB_SHA256SUM := d30b13f4ba2e3b6a2d4f020c0dee0a9fb9fc6fbcc2d561f36b78da4bf3802370 diff --git a/external/nss/ExternalProject_nss.mk b/external/nss/ExternalProject_nss.mk index acf418805772..daee76fc8fa9 100644 --- a/external/nss/ExternalProject_nss.mk +++ b/external/nss/ExternalProject_nss.mk @@ -34,7 +34,31 @@ $(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecuta ,nss) $(call gb_Trace_EndRange,nss,EXTERNAL) -else # OS!=WNT +else ifneq (,$(filter FREEBSD LINUX NETBSD OPENBSD SOLARIS,$(OS))) # non-WNT gyp-based + +# The nss build system uses 'python', so make it find our internal python if necessary. +nss_PYTHON := $(call gb_UnpackedTarball_get_dir,nss)/python +nss_SETUP_PYTHON := $(call gb_UnpackedTarball_get_dir,nss)/setup-python + +$(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecutable_get_dependencies,python) + $(call gb_Trace_StartRange,nss,EXTERNAL) + echo "#! /bin/sh" > $(nss_PYTHON) + pythondir=$$($(call gb_ExternalExecutable_get_command,python) -c 'import sys; import os; sys.stdout.write(os.path.dirname(sys.executable))') \ + && echo PATH=\"$$pythondir:\$$PATH\" >>$(nss_PYTHON) + echo '$(call gb_ExternalExecutable_get_command,python)' \"$$\@\" $(if $(ICECREAM_RUN), | sed 's/$(ICECREAM_RUN)//') >> $(nss_PYTHON) + chmod +x $(nss_PYTHON) + cp $(SRCDIR)/external/nss/setup-python $(nss_SETUP_PYTHON) + chmod +x $(nss_SETUP_PYTHON) + $(call gb_ExternalProject_run,build,\ + COMMA=$(COMMA) \ + PATH=$(call gb_UnpackedTarball_get_dir,nss):$(call gb_UnpackedTarball_get_dir,gyp):$$PATH \ + ./build.sh -v --disable-tests --enable-libpkix \ + $(if $(ENABLE_DBGUTIL),,--opt) \ + && rm -f $(call gb_UnpackedTarball_get_dir,nss)/dist/out/lib/*.a \ + ,nss) + $(call gb_Trace_EndRange,nss,EXTERNAL) + +else # platforms which need(?) the old build system (feel free to port to the new system) # make sure to specify NSPR_CONFIGURE_OPTS as env (before make command), so nss can append it's own defaults # OTOH specify e.g. CC and NSINSTALL as arguments (after make command), so they will overrule nss makefile values $(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecutable_get_dependencies,python) diff --git a/external/nss/Module_nss.mk b/external/nss/Module_nss.mk index eae9e9524480..be4b01e7ae4e 100644 --- a/external/nss/Module_nss.mk +++ b/external/nss/Module_nss.mk @@ -11,6 +11,7 @@ $(eval $(call gb_Module_Module,nss)) $(eval $(call gb_Module_add_targets,nss,\ UnpackedTarball_nss \ + UnpackedTarball_gyp \ ExternalPackage_nss \ ExternalProject_nss \ )) diff --git a/external/nss/UnpackedTarball_gyp.mk b/external/nss/UnpackedTarball_gyp.mk new file mode 100644 index 000000000000..50f7ba9c52b2 --- /dev/null +++ b/external/nss/UnpackedTarball_gyp.mk @@ -0,0 +1,18 @@ +# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +$(eval $(call gb_UnpackedTarball_UnpackedTarball,gyp)) + +$(eval $(call gb_UnpackedTarball_set_tarball,gyp,$(GYP_TARBALL))) + +$(eval $(call gb_UnpackedTarball_add_patches,gyp,\ + external/nss/gyp-buildsystem.patch.0 \ +)) + +# vim: set noet sw=4 ts=4: diff --git a/external/nss/UnpackedTarball_nss.mk b/external/nss/UnpackedTarball_nss.mk index 0e174879a702..4c6aa3db2030 100644 --- a/external/nss/UnpackedTarball_nss.mk +++ b/external/nss/UnpackedTarball_nss.mk @@ -34,6 +34,8 @@ $(eval $(call gb_UnpackedTarball_add_patches,nss,\ external/nss/nss-android.patch.1) \ external/nss/nss.bzmozilla1238154.patch \ external/nss/macos-dlopen.patch.0 \ + external/nss/buildsystem.patch.0 \ + external/nss/fix-cc-detect-with-ccache.patch.0 \ )) ifeq ($(COM_IS_CLANG),TRUE) diff --git a/external/nss/buildsystem.patch.0 b/external/nss/buildsystem.patch.0 new file mode 100644 index 000000000000..a27408d32669 --- /dev/null +++ b/external/nss/buildsystem.patch.0 @@ -0,0 +1,32 @@ +--- ./nss/coreconf/nspr.sh.sav 2019-11-19 20:55:30.000000000 +0100 ++++ ./nss/coreconf/nspr.sh 2020-02-19 22:25:05.863542364 +0100 +@@ -28,7 +28,7 @@ + + # These NSPR options are directory-specific, so they don't need to be + # included in nspr_opt and changing them doesn't force a rebuild of NSPR. +- extra_params=(--prefix="$dist_dir"/$target) ++ extra_params=(--prefix="$dist_dir"/out --libdir="$dist_dir"/out/lib --includedir="$dist_dir"/out/include) + if [ "$opt_build" = 1 ]; then + extra_params+=(--disable-debug --enable-optimize) + fi +--- ./nss/build.sh.sav 2019-11-19 20:55:30.000000000 +0100 ++++ ./nss/build.sh 2020-02-19 22:28:30.976068194 +0100 +@@ -243,10 +243,10 @@ + exit 3 + fi + # These extra arguments aren't used in determining whether to rebuild. +- obj_dir="$dist_dir/$target" ++ obj_dir="$dist_dir/out" + gyp_params+=(-Dnss_dist_obj_dir="$obj_dir") + if [ "$no_local_nspr" = 0 ]; then +- set_nspr_path "$obj_dir/include/nspr:$obj_dir/lib" ++ set_nspr_path "$obj_dir/include:$obj_dir/lib" + fi + + run_verbose run_scanbuild ${GYP} -f ninja "${gyp_params[@]}" "$cwd/nss.gyp" +@@ -263,4 +263,4 @@ + echo "Building NSS requires an installation of ninja: https://ninja-build.org/" 1>&2 + exit 3 + fi +-run_scanbuild "$ninja" -C "$target_dir" "${ninja_params[@]}" ++run_scanbuild setup-python "$ninja" -C "$target_dir" "${ninja_params[@]}" diff --git a/external/nss/fix-cc-detect-with-ccache.patch.0 b/external/nss/fix-cc-detect-with-ccache.patch.0 new file mode 100644 index 000000000000..6c3ebcfb77ed --- /dev/null +++ b/external/nss/fix-cc-detect-with-ccache.patch.0 @@ -0,0 +1,11 @@ +--- ./nss/coreconf/check_cc.py.sav 2020-03-20 17:58:57.275605539 +0100 ++++ ./nss/coreconf/check_cc.py 2020-03-20 18:10:56.053688945 +0100 +@@ -11,7 +11,7 @@ def main(): + cc = os.environ.get('CC', 'cc') + try: + cc_is_arg = sys.argv[1] in subprocess.check_output( +- [cc, '--version'], universal_newlines=True) ++ cc.split(' ') + ['--version'], universal_newlines=True) + except OSError: + # We probably just don't have CC/cc. + cc_is_arg = False diff --git a/external/nss/gyp-buildsystem.patch.0 b/external/nss/gyp-buildsystem.patch.0 new file mode 100644 index 000000000000..8323d1a7a7c3 --- /dev/null +++ b/external/nss/gyp-buildsystem.patch.0 @@ -0,0 +1,119 @@ +--- pylib/gyp/msvs_emulation.py.sav 2020-02-19 21:45:13.150161000 +0100 ++++ pylib/gyp/msvs_emulation.py 2020-02-21 23:51:56.815937600 +0100 +@@ -138,7 +138,7 @@ def _FindDirectXInstallation(): + if not dxsdk_dir: + # Setup params to pass to and attempt to launch reg.exe. + cmd = ['reg.exe', 'query', r'HKLM\Software\Microsoft\DirectX', '/s'] +- p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) + for line in p.communicate()[0].splitlines(): + if 'InstallPath' in line: + dxsdk_dir = line.split(' ')[3] + "\\" +@@ -321,7 +321,7 @@ class MsvsSettings(object): + # first level is globally for the configuration (this is what we consider + # "the" config at the gyp level, which will be something like 'Debug' or + # 'Release'), VS2015 and later only use this level +- if self.vs_version.short_name >= 2015: ++ if int(self.vs_version.short_name) >= 2015: + return config + # and a second target-specific configuration, which is an + # override for the global one. |config| is remapped here to take into +@@ -485,7 +485,7 @@ class MsvsSettings(object): + prefix='/arch:') + cflags.extend(['/FI' + f for f in self._Setting( + ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])]) +- if self.vs_version.project_version >= 12.0: ++ if float(self.vs_version.project_version) >= 12.0: + # New flag introduced in VS2013 (project version 12.0) Forces writes to + # the program database (PDB) to be serialized through MSPDBSRV.EXE. + # https://msdn.microsoft.com/en-us/library/dn502518.aspx +@@ -1050,7 +1050,7 @@ def GenerateEnvironmentFiles(toplevel_bu + args = vs.SetupScript(arch) + args.extend(('&&', 'set')) + popen = subprocess.Popen( +- args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + variables, _ = popen.communicate() + if popen.returncode != 0: + raise Exception('"%s" failed with error %d' % (args, popen.returncode)) +@@ -1071,7 +1071,7 @@ def GenerateEnvironmentFiles(toplevel_bu + args = vs.SetupScript(arch) + args.extend(('&&', + 'for', '%i', 'in', '(cl.exe)', 'do', '@echo', 'LOC:%~$PATH:i')) +- popen = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE) ++ popen = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE,universal_newlines=True) + output, _ = popen.communicate() + cl_paths[arch] = _ExtractCLPath(output) + return cl_paths +--- pylib/gyp/generator/ninja.py.sav 2020-02-19 21:45:13.150161000 +0100 ++++ pylib/gyp/generator/ninja.py 2020-02-22 00:42:48.090472000 +0100 +@@ -1741,7 +1741,10 @@ def CalculateGeneratorInputInfo(params): + def OpenOutput(path, mode='w'): + """Open |path| for writing, creating directories if necessary.""" + gyp.common.EnsureDirExists(path) +- return open(path, mode) ++ if sys.version_info[0] < 3: ++ return open(path, mode) ++ else: ++ return open(path, mode, encoding="utf-8") + + + def CommandWithWrapper(cmd, wrappers, prog): +--- pylib/gyp/win_tool.py.sav 2020-02-19 21:45:13.150161000 +0100 ++++ pylib/gyp/win_tool.py 2020-03-04 12:46:55.098189500 +0100 +@@ -130,7 +130,7 @@ class WinTool(object): + # For that reason, since going through the shell doesn't seem necessary on + # non-Windows don't do that there. + link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env, +- stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + out, _ = link.communicate() + for line in out.splitlines(): + if (not line.startswith(' Creating library ') and +@@ -197,8 +197,8 @@ class WinTool(object): + # and sometimes doesn't unfortunately. + with open(our_manifest, 'r') as our_f: + with open(assert_manifest, 'r') as assert_f: +- our_data = our_f.read().translate(None, string.whitespace) +- assert_data = assert_f.read().translate(None, string.whitespace) ++ our_data = our_f.read().translate(str.maketrans('','', string.whitespace)) ++ assert_data = assert_f.read().translate(str.maketrans('','', string.whitespace)) + if our_data != assert_data: + os.unlink(out) + def dump(filename): +@@ -223,7 +223,7 @@ class WinTool(object): + tool).""" + env = self._GetEnv(arch) + popen = subprocess.Popen(args, shell=True, env=env, +- stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + out, _ = popen.communicate() + for line in out.splitlines(): + if line and 'manifest authoring warning 81010002' not in line: +@@ -255,7 +255,7 @@ class WinTool(object): + idl] + env = self._GetEnv(arch) + popen = subprocess.Popen(args, shell=True, env=env, +- stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + out, _ = popen.communicate() + # Filter junk out of stdout, and write filtered versions. Output we want + # to filter is pairs of lines that look like this: +@@ -274,7 +274,7 @@ class WinTool(object): + """Filter logo banner from invocations of asm.exe.""" + env = self._GetEnv(arch) + popen = subprocess.Popen(args, shell=True, env=env, +- stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + out, _ = popen.communicate() + for line in out.splitlines(): + if (not line.startswith('Copyright (C) Microsoft Corporation') and +@@ -289,7 +289,7 @@ class WinTool(object): + don't support the /nologo flag.""" + env = self._GetEnv(arch) + popen = subprocess.Popen(args, shell=True, env=env, +- stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ++ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True) + out, _ = popen.communicate() + for line in out.splitlines(): + if (not line.startswith('Microsoft (R) Windows (R) Resource Compiler') and diff --git a/external/nss/setup-python b/external/nss/setup-python new file mode 100755 index 000000000000..7f4d367c8602 --- /dev/null +++ b/external/nss/setup-python @@ -0,0 +1,3 @@ +#! /bin/sh +# need to run using our python wrapper to set up python env. vars, for python calls from ninja +python -c "import sys; import subprocess; args=sys.argv; args.pop(0); args.pop(0); sys.exit(subprocess.call(args))" -- "$@" diff --git a/external/python3/ExternalPackage_python3.mk b/external/python3/ExternalPackage_python3.mk index 4dfc3b196f82..9036b6db4581 100644 --- a/external/python3/ExternalPackage_python3.mk +++ b/external/python3/ExternalPackage_python3.mk @@ -725,6 +725,7 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p Lib/multiprocessing/forkserver.py \ Lib/multiprocessing/heap.py \ Lib/multiprocessing/managers.py \ + Lib/multiprocessing/pool.py \ Lib/multiprocessing/popen_fork.py \ Lib/multiprocessing/popen_forkserver.py \ Lib/multiprocessing/popen_spawn_posix.py \ -- cgit v1.2.3