util/crossgcc: Add option to get packages from coreboot's mirror
coreboot has been keeping a mirror of all the toolchain packages used
for releases for quite a while now. This adds an option to fetch the
packages from the coreboot mirror directly to buildgcc.
This can help with both our releases and when one of the various
servers experiences interruptions or changes a path.
To do this, the URL and filename needed to be split apart, which led to
quite a few changes in the buildgcc script.
Signed-off-by: Martin Roth <gaumless@gmail.com>
Change-Id: I7df58dca152e7bfe9fde34d290e05b52515b20d9
Reviewed-on: https://review.coreboot.org/c/coreboot/+/70053
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Elyes Haouas <ehaouas@noos.fr>
Reviewed-by: Felix Singer <felixsinger@posteo.net>
diff --git a/util/crossgcc/Makefile.inc b/util/crossgcc/Makefile.inc
index a52fbf1..4534aaa 100644
--- a/util/crossgcc/Makefile.inc
+++ b/util/crossgcc/Makefile.inc
@@ -3,6 +3,7 @@
TOOLCHAIN_ARCHES := i386 x64 arm aarch64 riscv ppc64 nds32le
help_toolchain help::
+ @echo
@echo '*** Toolchain targets ***'
@echo ' crossgcc - Build coreboot cross-compilers for all platforms'
@echo ' crossgcc-clean - Remove all built coreboot cross-compilers'
@@ -14,6 +15,7 @@
@echo ' ARCH can be "$(subst $(spc),"$(comma) ",$(TOOLCHAIN_ARCHES))"'
@echo ' Use "make [target] CPUS=#" to build toolchain using multiple cores'
@echo ' Use "make [target] DEST=some/path" to install toolchain there'
+ @echo ' Use "make [target] BUILDGCC_OPTIONS="-m" to get packages from coreboot mirror"
@echo
# For the toolchain builds, use CPUS=x to use multiple processors to build
diff --git a/util/crossgcc/buildgcc b/util/crossgcc/buildgcc
index 6eeb6e7..d5466c1 100755
--- a/util/crossgcc/buildgcc
+++ b/util/crossgcc/buildgcc
@@ -30,6 +30,8 @@
SAVETEMPS=0
BOOTSTRAP=0
THREADS=1
+USE_COREBOOT_MIRROR=0
+COREBOOT_MIRROR_URL="https://www.coreboot.org/releases/crossgcc-sources"
# GCC toolchain version numbers
GMP_VERSION=6.2.1
@@ -43,29 +45,46 @@
CMAKE_VERSION=3.25.0
NASM_VERSION=2.15.05
-# GCC toolchain archive locations
-# These are sanitized by the jenkins toolchain test builder, so if
+# Filename for each package
+GMP_ARCHIVE="gmp-${GMP_VERSION}.tar.xz"
+MPFR_ARCHIVE="mpfr-${MPFR_VERSION}.tar.xz"
+MPC_ARCHIVE="mpc-${MPC_VERSION}.tar.gz"
+GCC_ARCHIVE="gcc-${GCC_VERSION}.tar.xz"
+BINUTILS_ARCHIVE="binutils-${BINUTILS_VERSION}.tar.xz"
+IASL_ARCHIVE="${IASL_VERSION}.tar.gz"
+# CLANG toolchain FILE locations
+LLVM_ARCHIVE="llvm-${CLANG_VERSION}.src.tar.xz"
+CLANG_ARCHIVE="clang-${CLANG_VERSION}.src.tar.xz"
+CRT_ARCHIVE="compiler-rt-${CLANG_VERSION}.src.tar.xz"
+CTE_ARCHIVE="clang-tools-extra-${CLANG_VERSION}.src.tar.xz"
+LLVMCMAKE_ARCHIVE="cmake-${CLANG_VERSION}.src.tar.xz"
+CMAKE_ARCHIVE="cmake-${CMAKE_VERSION}.tar.gz"
+NASM_ARCHIVE="nasm-${NASM_VERSION}.tar.bz2"
+
+# These URLs are sanitized by the jenkins toolchain test builder, so if
# a completely new URL is added here, it probably needs to be added
# to the jenkins build as well, or the builder won't download it.
-GMP_ARCHIVE="https://ftpmirror.gnu.org/gmp/gmp-${GMP_VERSION}.tar.xz"
-MPFR_ARCHIVE="https://ftpmirror.gnu.org/mpfr/mpfr-${MPFR_VERSION}.tar.xz"
-MPC_ARCHIVE="https://ftpmirror.gnu.org/mpc/mpc-${MPC_VERSION}.tar.gz"
-GCC_ARCHIVE="https://ftpmirror.gnu.org/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz"
-BINUTILS_ARCHIVE="https://ftpmirror.gnu.org/binutils/binutils-${BINUTILS_VERSION}.tar.xz"
-IASL_ARCHIVE="https://github.com/acpica/acpica/archive/refs/tags/${IASL_VERSION}.tar.gz"
-# CLANG toolchain archive locations
-LLVM_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/llvm-${CLANG_VERSION}.src.tar.xz"
-CLANG_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/clang-${CLANG_VERSION}.src.tar.xz"
-CRT_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/compiler-rt-${CLANG_VERSION}.src.tar.xz"
-CTE_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/clang-tools-extra-${CLANG_VERSION}.src.tar.xz"
-LLVMCMAKE_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/cmake-${CLANG_VERSION}.src.tar.xz"
-CMAKE_ARCHIVE="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}.tar.gz"
-NASM_ARCHIVE="https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2"
-ALL_ARCHIVES="$GMP_ARCHIVE $MPFR_ARCHIVE $MPC_ARCHIVE \
- $GCC_ARCHIVE $BINUTILS_ARCHIVE $IASL_ARCHIVE \
- $LLVM_ARCHIVE $CLANG_ARCHIVE $LLVMCMAKE_ARCHIVE \
- $CRT_ARCHIVE $CTE_ARCHIVE $CMAKE_ARCHIVE $NASM_ARCHIVE"
+# GCC toolchain archive locations
+GMP_BASE_URL="https://ftpmirror.gnu.org/gmp"
+MPFR_BASE_URL="https://ftpmirror.gnu.org/mpfr"
+MPC_BASE_URL="https://ftpmirror.gnu.org/mpc"
+GCC_BASE_URL="https://ftpmirror.gnu.org/gcc/gcc-${GCC_VERSION}"
+BINUTILS_BASE_URL="https://ftpmirror.gnu.org/binutils"
+IASL_BASE_URL="https://github.com/acpica/acpica/archive/refs/tags/"
+# CLANG toolchain archive locations
+LLVM_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
+CLANG_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
+CRT_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
+CTE_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
+LLVMCMAKE_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
+CMAKE_BASE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}"
+NASM_BASE_URL="https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}"
+
+ALL_ARCHIVES="$GMP_BASE_URL/$GMP_ARCHIVE $MPFR_BASE_URL/$MPFR_ARCHIVE $MPC_BASE_URL/$MPC_ARCHIVE \
+ $GCC_BASE_URL/$GCC_ARCHIVE $BINUTILS_BASE_URL/$BINUTILS_ARCHIVE $IASL_BASE_URL/$IASL_ARCHIVE \
+ $LLVM_BASE_URL/$LLVM_ARCHIVE $CLANG_BASE_URL/$CLANG_ARCHIVE $LLVMCMAKE_BASE_URL/$LLVMCMAKE_ARCHIVE \
+ $CRT_BASE_URL/$CRT_ARCHIVE $CTE_BASE_URL/$CTE_ARCHIVE $CMAKE_BASE_URL/$CMAKE_ARCHIVE $NASM_BASE_URL/$NASM_ARCHIVE"
# GCC toolchain directories
GMP_DIR="gmp-${GMP_VERSION}"
@@ -300,23 +319,27 @@
download() {
package=$1
# shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
+ if [ "${USE_COREBOOT_MIRROR}" -eq 0 ]; then
+ url="$(eval echo \$$package"_BASE_URL")"
+ else
+ url="${COREBOOT_MIRROR_URL}"
+ fi
- FILE=$(basename "$archive")
- printf " * $FILE "
+ file="$(eval echo \$$package"_ARCHIVE")"
+ printf " * ${file} "
- if test -f "tarballs/$FILE"; then
+ if test -f "tarballs/${file}"; then
printf "(cached)... "
else
- printf "(downloading from $archive)"
- rm -f "tarballs/$FILE"
+ printf "(downloading from ${url}/${file})"
+ rm -f "tarballs/${file}"
cd tarballs || exit 1
- download_showing_percentage "$archive"
+ download_showing_percentage "${url}/${file}"
cd ..
fi
- if [ ! -f "tarballs/$FILE" ]; then
- printf "${RED}Failed to download $FILE.${NC}\n"
+ if [ ! -f "tarballs/${file}" ]; then
+ printf "${RED}Failed to download ${file}.${NC}\n"
exit 1
fi
}
@@ -326,8 +349,7 @@
compute_hash() {
package=$1
# shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
- file="$(basename "$archive")"
+ file="$(eval echo \$$package"_ARCHIVE")"
if test -z "$CHECKSUM"; then
echo "${RED}\$CHECKSUM program missing. This is bad.${NC}" 1>&2
@@ -340,8 +362,7 @@
error_hash_missing() {
package="$1"
# shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
- file="$(basename "$archive")"
+ file="$(eval echo \$$package"_ARCHIVE")"
fullhashfile="util/crossgcc/sum/$file.cksum"
printf "${RED}hash file missing:${NC}\n\n" 1>&2
@@ -357,8 +378,7 @@
get_known_hash() {
package=$1
# shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
- file="$(basename "$archive")"
+ file="$(eval echo \$$package"_ARCHIVE")"
hashfile="sum/$file.cksum"
if [ ! -f "$hashfile" ]; then
@@ -377,8 +397,7 @@
known_hash="$2"
computed_hash="$3"
# shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
- file="$(basename "$archive")"
+ file="$(eval echo \$$package"_ARCHIVE")"
printf "${RED}hash mismatch:${NC}\n\n"
printf " expected (known) hash: $known_hash\n"
@@ -397,8 +416,6 @@
# hash; Bail out on mismatch or missing hash file.
verify_hash() {
package=$1
- # shellcheck disable=SC2086
- archive="$(eval echo \$$package"_ARCHIVE")"
known_hash="$(get_known_hash "$package")" || exit "$?"
computed_hash="$(compute_hash "$package")" || exit "$?"
@@ -418,7 +435,7 @@
# shellcheck disable=SC2086
dir="$(eval echo \$$package"_DIR")"
test -d "${dir}" && test -f "${dir}/.unpack_success" || (
- printf " * $(basename "$archive")\n"
+ printf " * "$archive"\n"
FLAGS=zxf
suffix=$(echo "$archive" | sed 's,.*\.,,')
if [ "$suffix" = "gz" ] && [ -n "$PIGZ" ]; then FLAGS="-I pigz -xf"
@@ -429,7 +446,7 @@
elif [ "$suffix" = "lzma" ]; then FLAGS="--lzma -xf"
fi
# shellcheck disable=SC2086
- $TAR $FLAGS "tarballs/$(basename "$archive")"
+ $TAR $FLAGS "tarballs/$archive"
for patch in patches/${dir}_*.patch; do
test -r "$patch" || continue
printf " o $(basename "$patch")\n"
@@ -915,12 +932,12 @@
getoptbrand="$(getopt -V 2>/dev/null | sed -e '1!d' -e 's,^\(......\).*,\1,')"
if [ "${getoptbrand}" = "getopt" ]; then
# Detected GNU getopt that supports long options.
- args=$(getopt -l print-version,version,help,clean,directory:,bootstrap,bootstrap-only,platform:,languages:,package:,jobs:,destdir:,savetemps,scripting,ccache,supported:,urls,nocolor -o WVhcd:bBp:l:P:j:D:tSys:un -- "$@")
+ args=$(getopt -l print-version,version,help,clean,directory:,bootstrap,bootstrap-only,platform:,languages:,package:,jobs:,destdir:,savetemps,scripting,ccache,supported:,urls,nocolor,mirror -o WVhcd:bBp:l:P:j:D:tSys:unm -- "$@")
getopt_ret=$?
eval set -- "$args"
else
# Detected non-GNU getopt
- args=$(getopt WVhcd:bBp:l:P:j:D:tSys:un $*)
+ args=$(getopt WVhcd:bBp:l:P:j:D:tSys:unm $*)
getopt_ret=$?
# shellcheck disable=SC2086
set -- $args
@@ -943,6 +960,7 @@
-B|--bootstrap-only) shift; BOOTSTRAPONLY=1; BOOTSTRAP=1;;
-p|--platform) shift; TARGETARCH="$1"; shift;;
-l|--languages) shift; LANGUAGES="$1"; shift;;
+ -m|--mirror) shift; USE_COREBOOT_MIRROR=1;;
-D|--destdir) shift; DESTDIR="$1"; shift;;
-j|--jobs) shift; THREADS="$1"; JOBS="-j $1"; shift;;
-P|--package) shift; PACKAGE="$1"; shift;;