add integrated cross-compile & qemu support This uses the existing CrOS toolchains & boards to cross-compile for arm and mipsel boards. It also includes qemu support for running tests. BUG=b/26741388 TEST=`make cross-arm all-libs` downloads & builds w/arm Change-Id: Ie974b1c3c2b61759e87f87bab535068a1c31a727 Reviewed-on: https://weave-review.googlesource.com/3000 Reviewed-by: Alex Vakulenko <avakulenko@google.com>
diff --git a/.gitignore b/.gitignore index 86c649b..c102afc 100644 --- a/.gitignore +++ b/.gitignore
@@ -1,5 +1,6 @@ *~ /out/ +/third_party/cross /third_party/include /third_party/lib /third_party/libevhtp
diff --git a/Makefile b/Makefile index 00f237c..8cbb0e7 100644 --- a/Makefile +++ b/Makefile
@@ -77,7 +77,7 @@ out/$(BUILD_MODE)/libweave.so : out/$(BUILD_MODE)/libweave_common.a $(CXX) -shared -Wl,-soname=libweave.so -o $@ -Wl,--whole-archive $^ -Wl,--no-whole-archive -lcrypto -lexpat -lpthread -lrt -include file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk +include cross.mk file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk ### # src/
diff --git a/README.md b/README.md index c640581..26f1cf7 100644 --- a/README.md +++ b/README.md
@@ -120,6 +120,24 @@ See [the examples README](/examples/daemon/README.md) for details. +### Cross-compiling + +The build supports transparently downloading & using a few cross-compilers. +Just add `cross-<arch>` to the command line in addition to the target you +want to actually build. + +This will cross-compile for an armv7 (hard float) target: + +``` +make cross-arm all-libs +``` + +This will cross-compile for a mips (little endian) target: + +``` +make cross-mipsel all-libs +``` + # Testing ### Run tests @@ -135,6 +153,16 @@ make testall ``` +### Cross-testing + +The build supports using qemu to run non-native tests. + +This will run armv7 tests through qemu: + +``` +make cross-arm testall +``` + # Making changes The [Android Developing site](https://source.android.com/source/developing.html)
diff --git a/cross.mk b/cross.mk new file mode 100644 index 0000000..e833ca9 --- /dev/null +++ b/cross.mk
@@ -0,0 +1,40 @@ +# Copyright 2016 The Weave Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Logic to easily run cross-compiling tests. + +CROSS_ROOT := $(PWD)/third_party/cross +CROSS_FLAGS := +DOWNLOAD_CROSS_TOOLCHAINS := no +QEMU_BASE := $(CROSS_ROOT)/app-emulation/qemu/usr/bin + +define cross-setup-gcc +.PHONY: $(1) +ifneq (,$$(findstring $(1),$$(MAKECMDGOALS))) +DOWNLOAD_CROSS_TOOLCHAINS := yes +CHOST := $(2) +BOARD := $(3) +CROSS := $$(CROSS_ROOT)/$$(CHOST)/bin/$$(CHOST)- +CC := $$(CROSS)gcc +CXX := $$(CROSS)g++ +AR := $$(CROSS)ar +CROSS_FLAGS += $(5) +QEMU := $$(QEMU_BASE)/$(4) -L $$(CROSS_ROOT)/$$(BOARD) +endif +endef + +# Whitespace matters with arguments, so we can't make this more readable :/. +$(eval $(call cross-setup-gcc,cross-arm,armv7a-cros-linux-gnueabi,arm-generic-full,qemu-arm,-mhard-float)) +$(eval $(call cross-setup-gcc,cross-mipsel,mipsel-cros-linux-gnu,mipsel-o32-generic-full,qemu-mipsel)) +$(eval $(call cross-setup-gcc,cross-x86,i686-pc-linux-gnu,x86-generic-full,qemu-i386)) +$(eval $(call cross-setup-gcc,cross-x86_64,x86_64-cros-linux-gnu,amd64-generic-full,qemu-x86_64)) + +ifeq ($(DOWNLOAD_CROSS_TOOLCHAINS),yes) +ifeq (,$(wildcard third_party/cross/$(BOARD))) +CROSS_FETCH_OUT := $(shell ./third_party/get_cross.sh >&2) +endif +CROSS_FLAGS += --sysroot $(CROSS_ROOT)/$(BOARD) +CC += $(CROSS_FLAGS) +CXX += $(CROSS_FLAGS) +endif
diff --git a/tests.mk b/tests.mk index d0042db..cd711ef 100644 --- a/tests.mk +++ b/tests.mk
@@ -12,6 +12,7 @@ ifeq (1, $(CLANG)) TEST_ENV += ASAN_SYMBOLIZER_PATH=$(shell which llvm-symbolizer-3.6) endif +TEST_ENV += $(QEMU) weave_test_obj_files := $(WEAVE_TEST_SRC_FILES:%.cc=out/$(BUILD_MODE)/%.o)
diff --git a/third_party/get_cross.sh b/third_party/get_cross.sh new file mode 100755 index 0000000..f69291c --- /dev/null +++ b/third_party/get_cross.sh
@@ -0,0 +1,186 @@ +#!/bin/bash +# Copyright 2016 The Weave Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +set -e + +SCRIPT=$(readlink -f "$0") +THIRD_PARTY=$(dirname "${SCRIPT}") +cd "${THIRD_PARTY}" + +OUT="cross" +DISTDIR="${OUT}/distfiles" + +CROS_OVERLAY_URL="https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay/+/master/chromeos" +CONF_SDK_LATEST="${CROS_OVERLAY_URL}/binhost/host/sdk_version.conf" + +SDK_BUCKET="https://commondatastorage.googleapis.com/chromiumos-sdk" +BINPKG_BUCKET="https://commondatastorage.googleapis.com/chromeos-prebuilt" +CROS_BUCKET="https://commondatastorage.googleapis.com/chromeos-image-archive" + +PKGS=( + app-emulation/qemu +) +TARGETS=( +# aarch64-cros-linux-gnu + armv7a-cros-linux-gnueabi + i686-pc-linux-gnu +# mips-cros-linux-gnu + mipsel-cros-linux-gnu + x86_64-cros-linux-gnu +) +BOARDS=( +# aarch64-generic-full + amd64-generic-full + arm-generic-full +# mips-o32-generic-full + mipsel-o32-generic-full + x86-generic-full +) + +usage() { + cat <<EOF +Usage: get_cross.sh + +Download cross-compilers for building & testing against other arches. +EOF + exit 0 +} + +get_gitiles() { + local url="$1" data + data=$(curl -s "${url}?format=TEXT") + echo "${data}" | base64 -d +} + +json() { + local file="$1" arg="$2" + python <<EOF +import json +print(json.load(open("${file}"))${arg}) +EOF +} + +fetch() { + local url=$1 + file="${2:-${DISTDIR}/${url##*/}}" + if [[ ! -e ${file} ]]; then + printf '[downloading] ' + mkdir -p "${DISTDIR}" + wget "${url}" -O "${file}" + fi +} + +unpack() { + local out="$1" file="$2" + printf '[unpacking] ' + rm -rf "${out}" + mkdir -p "${out}" + tar xf "${file}" -C "${out}" +} + +fetch_pkgs() { + local pkg + local sub_url url file manifest + local out ver_file old_ver ver + + # Grab a few helper packages. + printf 'Getting SDK manifest ... ' + sub_url="cros-sdk-${SDK_LATEST_VERSION}.tar.xz.Manifest" + url="${SDK_BUCKET}/${sub_url}" + fetch "${url}" + manifest=${file} + printf '%s\n' "${manifest}" + + for pkg in "${PKGS[@]}"; do + printf 'Getting binpkg %s ... ' "${pkg}" + ver=$(json "${manifest}" '["packages"]["app-emulation/qemu"][0][0]') + sub_url="host/amd64/amd64-host/chroot-${SDK_LATEST_VERSION}/packages/${pkg}-${ver}.tbz2" + url="${BINPKG_BUCKET}/${sub_url}" + fetch "${url}" + + out="${OUT}/${pkg}" + ver_file="${out}/.ver" + old_ver=$(cat "${ver_file}" 2>/dev/null || :) + if [[ "${old_ver}" != "${ver}" ]]; then + unpack "${out}" "${file}" + echo "${ver}" > "${ver_file}" + fi + + printf '%s\n' "${ver}" + done +} + +fetch_toolchains() { + local target + local sub_url url file + local out ver_file ver + + # Download the base toolchains. + for target in "${TARGETS[@]}"; do + printf 'Getting toolchain for %s ... ' "${target}" + + sub_url="${TC_PATH/\%(target)s/${target}}" + url="${SDK_BUCKET}/${sub_url}" + file="${DISTDIR}/${url##*/}" + fetch "${url}" + + out="${OUT}/${target}" + ver_file="${out}/.ver" + ver=$(cat "${ver_file}" 2>/dev/null || :) + if [[ "${ver}" != "${SDK_LATEST_VERSION}" ]]; then + unpack "${out}" "${file}" + echo "${SDK_LATEST_VERSION}" > "${ver_file}" + fi + + printf '%s\n' "${sub_url}" + done +} + +fetch_sysroots() { + local board + local board_latest_url sub_url url file + local out ver_file ver + + # Get the full sysroot. + for board in "${BOARDS[@]}"; do + printf 'Getting sysroot for %s ... ' "${board}" + board_latest_url="${CROS_BUCKET}/${board}/LATEST-master" + if ! board_ver=$(curl --fail -s "${board_latest_url}"); then + echo 'error: not found' + continue + fi + + url="${CROS_BUCKET}/${board}/${board_ver}/sysroot_chromeos-base_chromeos-chrome.tar.xz" + file="${DISTDIR}/${board}-${board_ver}-${url##*/}" + fetch "${url}" "${file}" + + out="${OUT}/${board}" + ver_file="${out}/.ver" + ver=$(cat "${ver_file}" 2>/dev/null || :) + if [[ "${ver}" != "${board_ver}" ]]; then + unpack "${out}" "${file}" + echo "${board_ver}" > "${ver_file}" + fi + + printf '%s\n' "${board_ver}" + done +} + +main() { + if [[ $# -ne 0 ]]; then + usage + fi + + # Get the current SDK versions. + printf 'Getting CrOS SDK version ... ' + data=$(get_gitiles "${CONF_SDK_LATEST}") + eval "${data}" + echo "${SDK_LATEST_VERSION}" + + fetch_pkgs + fetch_toolchains + fetch_sysroots +} +main "$@"