add integrated cross-compile & qemu support
This uses the existing CrOS toolchains & boards to cross-compile for arm
and mipsel boards. It also includes qemu support for running tests.
BUG=b/26741388
TEST=`make cross-arm all-libs` downloads & builds w/arm
Change-Id: Ie974b1c3c2b61759e87f87bab535068a1c31a727
Reviewed-on: https://weave-review.googlesource.com/3000
Reviewed-by: Alex Vakulenko <avakulenko@google.com>
diff --git a/.gitignore b/.gitignore
index 86c649b..c102afc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
*~
/out/
+/third_party/cross
/third_party/include
/third_party/lib
/third_party/libevhtp
diff --git a/Makefile b/Makefile
index 00f237c..8cbb0e7 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,7 @@
out/$(BUILD_MODE)/libweave.so : out/$(BUILD_MODE)/libweave_common.a
$(CXX) -shared -Wl,-soname=libweave.so -o $@ -Wl,--whole-archive $^ -Wl,--no-whole-archive -lcrypto -lexpat -lpthread -lrt
-include file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk
+include cross.mk file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk
###
# src/
diff --git a/README.md b/README.md
index c640581..26f1cf7 100644
--- a/README.md
+++ b/README.md
@@ -120,6 +120,24 @@
See [the examples README](/examples/daemon/README.md) for details.
+### Cross-compiling
+
+The build supports transparently downloading & using a few cross-compilers.
+Just add `cross-<arch>` to the command line in addition to the target you
+want to actually build.
+
+This will cross-compile for an armv7 (hard float) target:
+
+```
+make cross-arm all-libs
+```
+
+This will cross-compile for a mips (little endian) target:
+
+```
+make cross-mipsel all-libs
+```
+
# Testing
### Run tests
@@ -135,6 +153,16 @@
make testall
```
+### Cross-testing
+
+The build supports using qemu to run non-native tests.
+
+This will run armv7 tests through qemu:
+
+```
+make cross-arm testall
+```
+
# Making changes
The [Android Developing site](https://source.android.com/source/developing.html)
diff --git a/cross.mk b/cross.mk
new file mode 100644
index 0000000..e833ca9
--- /dev/null
+++ b/cross.mk
@@ -0,0 +1,40 @@
+# Copyright 2016 The Weave Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Logic to easily run cross-compiling tests.
+
+CROSS_ROOT := $(PWD)/third_party/cross
+CROSS_FLAGS :=
+DOWNLOAD_CROSS_TOOLCHAINS := no
+QEMU_BASE := $(CROSS_ROOT)/app-emulation/qemu/usr/bin
+
+define cross-setup-gcc
+.PHONY: $(1)
+ifneq (,$$(findstring $(1),$$(MAKECMDGOALS)))
+DOWNLOAD_CROSS_TOOLCHAINS := yes
+CHOST := $(2)
+BOARD := $(3)
+CROSS := $$(CROSS_ROOT)/$$(CHOST)/bin/$$(CHOST)-
+CC := $$(CROSS)gcc
+CXX := $$(CROSS)g++
+AR := $$(CROSS)ar
+CROSS_FLAGS += $(5)
+QEMU := $$(QEMU_BASE)/$(4) -L $$(CROSS_ROOT)/$$(BOARD)
+endif
+endef
+
+# Whitespace matters with arguments, so we can't make this more readable :/.
+$(eval $(call cross-setup-gcc,cross-arm,armv7a-cros-linux-gnueabi,arm-generic-full,qemu-arm,-mhard-float))
+$(eval $(call cross-setup-gcc,cross-mipsel,mipsel-cros-linux-gnu,mipsel-o32-generic-full,qemu-mipsel))
+$(eval $(call cross-setup-gcc,cross-x86,i686-pc-linux-gnu,x86-generic-full,qemu-i386))
+$(eval $(call cross-setup-gcc,cross-x86_64,x86_64-cros-linux-gnu,amd64-generic-full,qemu-x86_64))
+
+ifeq ($(DOWNLOAD_CROSS_TOOLCHAINS),yes)
+ifeq (,$(wildcard third_party/cross/$(BOARD)))
+CROSS_FETCH_OUT := $(shell ./third_party/get_cross.sh >&2)
+endif
+CROSS_FLAGS += --sysroot $(CROSS_ROOT)/$(BOARD)
+CC += $(CROSS_FLAGS)
+CXX += $(CROSS_FLAGS)
+endif
diff --git a/tests.mk b/tests.mk
index d0042db..cd711ef 100644
--- a/tests.mk
+++ b/tests.mk
@@ -12,6 +12,7 @@
ifeq (1, $(CLANG))
TEST_ENV += ASAN_SYMBOLIZER_PATH=$(shell which llvm-symbolizer-3.6)
endif
+TEST_ENV += $(QEMU)
weave_test_obj_files := $(WEAVE_TEST_SRC_FILES:%.cc=out/$(BUILD_MODE)/%.o)
diff --git a/third_party/get_cross.sh b/third_party/get_cross.sh
new file mode 100755
index 0000000..f69291c
--- /dev/null
+++ b/third_party/get_cross.sh
@@ -0,0 +1,186 @@
+#!/bin/bash
+# Copyright 2016 The Weave Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+SCRIPT=$(readlink -f "$0")
+THIRD_PARTY=$(dirname "${SCRIPT}")
+cd "${THIRD_PARTY}"
+
+OUT="cross"
+DISTDIR="${OUT}/distfiles"
+
+CROS_OVERLAY_URL="https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay/+/master/chromeos"
+CONF_SDK_LATEST="${CROS_OVERLAY_URL}/binhost/host/sdk_version.conf"
+
+SDK_BUCKET="https://commondatastorage.googleapis.com/chromiumos-sdk"
+BINPKG_BUCKET="https://commondatastorage.googleapis.com/chromeos-prebuilt"
+CROS_BUCKET="https://commondatastorage.googleapis.com/chromeos-image-archive"
+
+PKGS=(
+ app-emulation/qemu
+)
+TARGETS=(
+# aarch64-cros-linux-gnu
+ armv7a-cros-linux-gnueabi
+ i686-pc-linux-gnu
+# mips-cros-linux-gnu
+ mipsel-cros-linux-gnu
+ x86_64-cros-linux-gnu
+)
+BOARDS=(
+# aarch64-generic-full
+ amd64-generic-full
+ arm-generic-full
+# mips-o32-generic-full
+ mipsel-o32-generic-full
+ x86-generic-full
+)
+
+usage() {
+ cat <<EOF
+Usage: get_cross.sh
+
+Download cross-compilers for building & testing against other arches.
+EOF
+ exit 0
+}
+
+get_gitiles() {
+ local url="$1" data
+ data=$(curl -s "${url}?format=TEXT")
+ echo "${data}" | base64 -d
+}
+
+json() {
+ local file="$1" arg="$2"
+ python <<EOF
+import json
+print(json.load(open("${file}"))${arg})
+EOF
+}
+
+fetch() {
+ local url=$1
+ file="${2:-${DISTDIR}/${url##*/}}"
+ if [[ ! -e ${file} ]]; then
+ printf '[downloading] '
+ mkdir -p "${DISTDIR}"
+ wget "${url}" -O "${file}"
+ fi
+}
+
+unpack() {
+ local out="$1" file="$2"
+ printf '[unpacking] '
+ rm -rf "${out}"
+ mkdir -p "${out}"
+ tar xf "${file}" -C "${out}"
+}
+
+fetch_pkgs() {
+ local pkg
+ local sub_url url file manifest
+ local out ver_file old_ver ver
+
+ # Grab a few helper packages.
+ printf 'Getting SDK manifest ... '
+ sub_url="cros-sdk-${SDK_LATEST_VERSION}.tar.xz.Manifest"
+ url="${SDK_BUCKET}/${sub_url}"
+ fetch "${url}"
+ manifest=${file}
+ printf '%s\n' "${manifest}"
+
+ for pkg in "${PKGS[@]}"; do
+ printf 'Getting binpkg %s ... ' "${pkg}"
+ ver=$(json "${manifest}" '["packages"]["app-emulation/qemu"][0][0]')
+ sub_url="host/amd64/amd64-host/chroot-${SDK_LATEST_VERSION}/packages/${pkg}-${ver}.tbz2"
+ url="${BINPKG_BUCKET}/${sub_url}"
+ fetch "${url}"
+
+ out="${OUT}/${pkg}"
+ ver_file="${out}/.ver"
+ old_ver=$(cat "${ver_file}" 2>/dev/null || :)
+ if [[ "${old_ver}" != "${ver}" ]]; then
+ unpack "${out}" "${file}"
+ echo "${ver}" > "${ver_file}"
+ fi
+
+ printf '%s\n' "${ver}"
+ done
+}
+
+fetch_toolchains() {
+ local target
+ local sub_url url file
+ local out ver_file ver
+
+ # Download the base toolchains.
+ for target in "${TARGETS[@]}"; do
+ printf 'Getting toolchain for %s ... ' "${target}"
+
+ sub_url="${TC_PATH/\%(target)s/${target}}"
+ url="${SDK_BUCKET}/${sub_url}"
+ file="${DISTDIR}/${url##*/}"
+ fetch "${url}"
+
+ out="${OUT}/${target}"
+ ver_file="${out}/.ver"
+ ver=$(cat "${ver_file}" 2>/dev/null || :)
+ if [[ "${ver}" != "${SDK_LATEST_VERSION}" ]]; then
+ unpack "${out}" "${file}"
+ echo "${SDK_LATEST_VERSION}" > "${ver_file}"
+ fi
+
+ printf '%s\n' "${sub_url}"
+ done
+}
+
+fetch_sysroots() {
+ local board
+ local board_latest_url sub_url url file
+ local out ver_file ver
+
+ # Get the full sysroot.
+ for board in "${BOARDS[@]}"; do
+ printf 'Getting sysroot for %s ... ' "${board}"
+ board_latest_url="${CROS_BUCKET}/${board}/LATEST-master"
+ if ! board_ver=$(curl --fail -s "${board_latest_url}"); then
+ echo 'error: not found'
+ continue
+ fi
+
+ url="${CROS_BUCKET}/${board}/${board_ver}/sysroot_chromeos-base_chromeos-chrome.tar.xz"
+ file="${DISTDIR}/${board}-${board_ver}-${url##*/}"
+ fetch "${url}" "${file}"
+
+ out="${OUT}/${board}"
+ ver_file="${out}/.ver"
+ ver=$(cat "${ver_file}" 2>/dev/null || :)
+ if [[ "${ver}" != "${board_ver}" ]]; then
+ unpack "${out}" "${file}"
+ echo "${board_ver}" > "${ver_file}"
+ fi
+
+ printf '%s\n' "${board_ver}"
+ done
+}
+
+main() {
+ if [[ $# -ne 0 ]]; then
+ usage
+ fi
+
+ # Get the current SDK versions.
+ printf 'Getting CrOS SDK version ... '
+ data=$(get_gitiles "${CONF_SDK_LATEST}")
+ eval "${data}"
+ echo "${SDK_LATEST_VERSION}"
+
+ fetch_pkgs
+ fetch_toolchains
+ fetch_sysroots
+}
+main "$@"
