10568660ac
The logic that sets up the QEMU static binary to permit running alien binary formats is quite complex. This commit cleans up a lot of the logic around this and adds a huge number of comments to try and clarify why things happen when they do. Ultimately this commit provides a lot of the ground work to factor out all of the chroot setup into a seperate shell file that can be sourced by other mklive tasks that need to operate inside of a chroot with a possibly alien executable format.
398 lines
16 KiB
Bash
398 lines
16 KiB
Bash
#!/bin/sh
|
|
#-
|
|
# Copyright (c) 2013-2015 Juan Romero Pardines.
|
|
# Copyright (c) 2017 Google
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#-
|
|
|
|
readonly PROGNAME=$(basename "$0")
|
|
readonly ARCH=$(uname -m)
|
|
|
|
trap 'die "Interrupted! exiting..."' INT TERM HUP
|
|
|
|
|
|
info_msg() {
|
|
printf "\033[1m%s\n\033[m" "$@"
|
|
}
|
|
|
|
die() {
|
|
printf "FATAL: %s\n" "$@"
|
|
umount_pseudofs
|
|
[ -d "$ROOTFS" ] && rm -rf "$ROOTFS"
|
|
exit 1
|
|
}
|
|
|
|
|
|
# Even though we only support really one target for most of these
|
|
# architectures this lets us refer to these quickly and easily by
|
|
# XBPS_ARCH. This makes it a lot more obvious what is happening later
|
|
# in the script, and it makes it easier to consume the contents of
|
|
# these down the road in later scripts.
|
|
usage() {
|
|
cat <<_EOF
|
|
Usage: $PROGNAME [options] <arch>
|
|
|
|
Supported architectures: i686, i686-musl, x86_64, x86_64-musl,
|
|
armv5tel, armv5tel-musl, armv6l, armv6l-musl, armv7l, armv7l-musl
|
|
aarch64, aarch64-musl,
|
|
mipsel, mipsel-musl
|
|
|
|
|
|
Options
|
|
-b <syspkg> Set an alternative base-system package (defaults to base-system)
|
|
-c <dir> Set XBPS cache directory (defaults to \$PWD/xbps-cachedir-<arch>)
|
|
-C <file> Full path to the XBPS configuration file
|
|
-h Show this help
|
|
-r <repo> Set XBPS repository (may be set multiple times)
|
|
-V Show version
|
|
_EOF
|
|
}
|
|
|
|
mount_pseudofs() {
|
|
# This function ensures that the psuedofs mountpoints are present
|
|
# in the chroot. Strictly they are not necessary to have for many
|
|
# commands, but bind-mounts are cheap and it isn't too bad to just
|
|
# mount them all the time.
|
|
for f in dev proc sys; do
|
|
# In a naked chroot there is nothing to bind the mounts to, so
|
|
# we need to create directories for these first.
|
|
[ ! -d "$ROOTFS/$f" ] && mkdir -p "$ROOTFS/$f"
|
|
if ! mountpoint -q "$ROOTFS/$f" ; then
|
|
# It is VERY important that this only happen if the
|
|
# pseudofs isn't already mounted. If it already is then
|
|
# this is virtually impossible to troubleshoot because it
|
|
# looks like the subsequent umount just isn't working.
|
|
mount -r --bind /$f "$ROOTFS/$f"
|
|
fi
|
|
done
|
|
}
|
|
|
|
umount_pseudofs() {
|
|
# This function cleans up the mounts in the chroot. Failure to
|
|
# clean up these mounts will prevent the tmpdir from being
|
|
# deletable instead throwing the error "Device or Resource Busy".
|
|
# The '-f' option is passed to umount to account for the
|
|
# contingency where the psuedofs mounts are not present.
|
|
if [ -d "${ROOTFS}" ]; then
|
|
for f in dev proc sys; do
|
|
umount -f "$ROOTFS/$f" >/dev/null 2>&1
|
|
done
|
|
fi
|
|
}
|
|
|
|
run_cmd_target() {
|
|
info_msg "Running $* for target $XBPS_TARGET_ARCH ..."
|
|
if [ "$XBPS_TARGET_ARCH" = "$(xbps-uhelper arch)" ] ; then
|
|
# This is being run on the same architecture as the host,
|
|
# therefore we should set XBPS_ARCH.
|
|
if ! eval XBPS_ARCH="$XBPS_TARGET_ARCH" "$@" ; then
|
|
die "Could not run command $*"
|
|
fi
|
|
else
|
|
# This is being run on a foriegn arch, therefore we should set
|
|
# XBPS_TARGET_ARCH. In this case XBPS will not attempt
|
|
# certain actions and will require reconfiguration later.
|
|
if ! eval XBPS_TARGET_ARCH="$XBPS_TARGET_ARCH" "$@" ; then
|
|
die "Could not run command $*"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
run_cmd() {
|
|
info_msg "Running $*"
|
|
eval "$@"
|
|
}
|
|
|
|
run_cmd_chroot() {
|
|
# General purpose chroot function which makes sure the chroot is
|
|
# prepared. This function takes 2 arguments, the location to
|
|
# chroot to and the command to run.
|
|
|
|
# This is an idempotent function, it is safe to call every time
|
|
# before entering the chroot. This has the advantage of making
|
|
# execution in the chroot appear as though it "Just Works(tm)".
|
|
register_binfmt
|
|
|
|
# Before we step into the chroot we need to make sure the
|
|
# pseudo-filesystems are ready to go. Not all commands will need
|
|
# this, but its still a good idea to call it here anyway.
|
|
mount_pseudofs
|
|
|
|
# With assurance that things will run now we can jump into the
|
|
# chroot and run stuff!
|
|
chroot "$1" sh -c "$2"
|
|
}
|
|
|
|
cleanup_chroot() {
|
|
# This function cleans up the chroot shims that are used by QEMU
|
|
# to allow builds on alien platforms. It takes no arguments but
|
|
# expects the global $ROOTFS variable to be set.
|
|
|
|
# Un-Mount the pseudofs mounts if they were mounted
|
|
umount_pseudofs
|
|
|
|
# If a QEMU binary was copied in, remove that as well
|
|
if [ -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then
|
|
rm "$ROOTFS/usr/bin/$QEMU_BIN"
|
|
fi
|
|
}
|
|
|
|
# TODO: Figure out how to register the binfmt for x86_64 and for i686
|
|
# to facilitate building on alien build systems.
|
|
register_binfmt() {
|
|
# This function sets up everything that is needed to be able to
|
|
# chroot into a ROOTFS and be able to run commands there. This
|
|
# really matters on platforms where the host architecture is
|
|
# different from the target, and you wouldn't be able to run
|
|
# things like xbps-reconfigure -a. This function is idempotent
|
|
# (You can run it multiple times without modifying state). This
|
|
# function takes no arguments, but does expect the global variable
|
|
# $XBPS_TARGET_ARCH to be set.
|
|
|
|
# This select sets up the "magic" bytes in /proc that let the
|
|
# kernel select an alternate interpreter. More values for this
|
|
# map can be obtained from here:
|
|
# https://github.com/qemu/qemu/blob/master/scripts/qemu-binfmt-conf.sh
|
|
case "${XBPS_TARGET_ARCH}" in
|
|
armv*)
|
|
_cpu=arm
|
|
_magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00"
|
|
_mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
|
|
QEMU_BIN=qemu-arm-static
|
|
;;
|
|
aarch64*)
|
|
_cpu=aarch64
|
|
_magic="\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7"
|
|
_mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff"
|
|
QEMU_BIN=qemu-aarch64-static
|
|
;;
|
|
mipsel*)
|
|
_cpu=mipsel
|
|
_magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00"
|
|
_mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
|
|
QEMU_BIN=qemu-mipsel-static
|
|
;;
|
|
*86*)
|
|
info_msg "FIXME: Assuming that x86 instructions are native"
|
|
QEMU_BIN=NATIVE
|
|
;;
|
|
*)
|
|
die "Unknown target architecture!"
|
|
;;
|
|
esac
|
|
|
|
# In the special case where the build is native we can return
|
|
# without doing anything else
|
|
if [ "$QEMU_BIN" = "NATIVE" ] ; then
|
|
return
|
|
fi
|
|
|
|
# For builds that do not match the host architecture, the correct
|
|
# qemu binary will be required.
|
|
if ! $QEMU_BIN -version >/dev/null 2>&1; then
|
|
die "$QEMU_BIN binary is missing in your system, exiting."
|
|
fi
|
|
|
|
# In order to use the binfmt system the binfmt_misc mountpoint
|
|
# must exist inside of proc
|
|
if ! mountpoint -q /proc/sys/fs/binfmt_misc ; then
|
|
modprobe -q binfmt_misc
|
|
mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null
|
|
fi
|
|
|
|
# Only register if the map is incomplete
|
|
if [ ! -f /proc/sys/fs/binfmt_misc/qemu-$_cpu ] ; then
|
|
echo ":qemu-$_cpu:M::$_magic:$_mask:$QEMU_BIN:" > /proc/sys/fs/binfmt_misc/register 2>/dev/null
|
|
fi
|
|
|
|
# If the static binary isn't in the chroot then the chroot will
|
|
# fail. The kernel knows about the map but without the static
|
|
# version there's no interpreter in the chroot, only the
|
|
# dynamically linked one in the host. To simplify things we just
|
|
# use the static one always and make sure it shows up at the same
|
|
# place in the host and the chroot.
|
|
if [ ! -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then
|
|
cp -f "$(which "$QEMU_BIN")" "$ROOTFS/usr/bin" ||
|
|
die "Could not install $QEMU_BIN to $ROOTFS/usr/bin/"
|
|
fi
|
|
}
|
|
|
|
#
|
|
# main()
|
|
#
|
|
while getopts "C:c:h:r:V" opt; do
|
|
case $opt in
|
|
C) XBPS_CONFFILE="-C $OPTARG";;
|
|
c) XBPS_CACHEDIR="--cachedir=$OPTARG";;
|
|
h) usage; exit 0;;
|
|
r) XBPS_REPOSITORY="$XBPS_REPOSITORY --repository=$OPTARG";;
|
|
V) echo "$PROGNAME @@MKLIVE_VERSION@@"; exit 0;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
XBPS_TARGET_ARCH="$1"
|
|
|
|
# This is an aweful hack since the script isn't using privesc
|
|
# mechanisms selectively. This is a TODO item.
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
die "need root perms to continue, exiting."
|
|
fi
|
|
|
|
# If the arch wasn't set let's bail out now, nothing else in this
|
|
# script will work without knowing what we're trying to build for.
|
|
if [ -z "$XBPS_TARGET_ARCH" ]; then
|
|
echo "$PROGNAME: arch was not set!"
|
|
usage; exit 1
|
|
fi
|
|
|
|
# If the repository hasn't already been set, we set it to a sane value
|
|
# here. These should all resolve even if they won't have the
|
|
# appropriate repodata files for the selected architecture.
|
|
: "${XBPS_REPOSITORY:=--repository=http://repo.voidlinux.eu/current \
|
|
--repository=http://repo.voidlinux.eu/current/musl \
|
|
--repository=http://repo.voidlinux.eu/current/aarch64}"
|
|
|
|
# The package artifacts are cacheable, but they need to be isolated
|
|
# from the host cache.
|
|
: "${XBPS_CACHEDIR:=--cachedir=$PWD/xbps-cache/${XBPS_TARGET_ARCH}}"
|
|
|
|
# The following binaries are required to proceed
|
|
for f in chroot tar xbps-install xbps-reconfigure xbps-query; do
|
|
if ! which $f >/dev/null ; then
|
|
die "$f binary is missing in your system, exiting."
|
|
fi
|
|
done
|
|
|
|
# We need to operate on a tempdir, if this fails to create, it is
|
|
# absolutely crucial to bail out so that we don't hose the system that
|
|
# is running the script.
|
|
ROOTFS=$(mktemp -d) || die "failed to create tempdir, exiting..."
|
|
|
|
# This maintains the chain of trust, the keys in the repo are known to
|
|
# be good and so we copy those. Why don't we just use the ones on the
|
|
# host system? That's a good point, but there's no promise that the
|
|
# system running the script is Void, or that those keys haven't been
|
|
# tampered with. Its much easier to use these since the will always
|
|
# exist.
|
|
mkdir -p "$ROOTFS/var/db/xbps/keys"
|
|
cp keys/*.plist "$ROOTFS/var/db/xbps/keys"
|
|
|
|
# This sets up files that are important for XBPS to work on the new
|
|
# filesystem. It does not actually install anything.
|
|
run_cmd_target "xbps-install -S $XBPS_CONFFILE $XBPS_CACHEDIR $XBPS_REPOSITORY -r $ROOTFS"
|
|
|
|
# Later scripts expect the permissions on / to be the canonical 755,
|
|
# so we set this here.
|
|
chmod 755 "$ROOTFS"
|
|
|
|
# The pseudofs mountpoints are needed for the qemu support in cases
|
|
# where we are running things that aren't natively executable.
|
|
mount_pseudofs
|
|
|
|
# With everything setup, we can now run the install to load the
|
|
# base-voidstrap package into the rootfs. This will not produce a
|
|
# bootable system but will instead produce a base component that can
|
|
# be quickly expanded to perform other actions on.
|
|
run_cmd_target "xbps-install -S $XBPS_CONFFILE $XBPS_CACHEDIR $XBPS_REPOSITORY -r $ROOTFS -y base-voidstrap"
|
|
|
|
# Enable en_US.UTF-8 locale and generate it into the target ROOTFS.
|
|
# This is a bit of a hack since some glibc stuff doesn't really work
|
|
# correctly without a locale being generated. While some could argue
|
|
# that this is an arbitrary or naive choice to enable the en_US
|
|
# locale, most people using Void are able to work with the English
|
|
# language at least enough to enable thier preferred locale. If this
|
|
# truly becomes an issue in the future this hack can be revisited.
|
|
if [ -e "$ROOTFS/etc/default/libc-locales" ]; then
|
|
LOCALE=en_US.UTF-8
|
|
sed -e "s/\#\(${LOCALE}.*\)/\1/g" -i "$ROOTFS/etc/default/libc-locales"
|
|
fi
|
|
|
|
# The reconfigure step needs to execute code that's been compiled for
|
|
# the target architecture. Since the target isn't garanteed to be the
|
|
# same as the host, this needs to be done via qemu.
|
|
info_msg "Reconfiguring packages for ${XBPS_TARGET_ARCH} ..."
|
|
case "$XBPS_TARGET_ARCH" in
|
|
# TODO: Rather than asserting that x86 code will work, check
|
|
# instead if the system that is hosting this script is the same as
|
|
# the target, using binfmt if it is not.
|
|
i686*|x86_64*)
|
|
run_cmd "XBPS_ARCH=${XBPS_TARGET_ARCH} xbps-reconfigure -r $ROOTFS base-files"
|
|
;;
|
|
*)
|
|
# This step sets up enough of the base-files that the chroot
|
|
# will work and they can be reconfigured natively. Without
|
|
# this step there isn't enough configured for ld to work.
|
|
# This step runs as the host architecture.
|
|
run_cmd "xbps-reconfigure -r $ROOTFS base-files"
|
|
|
|
# Now running as the target system, this step reconfigures the
|
|
# base-files completely. Certain things just won't work in
|
|
# the first pass, so this cleans up any issues that linger.
|
|
run_cmd_chroot "$ROOTFS" "env -i xbps-reconfigure -f base-files"
|
|
|
|
# TODO: determine why these lines are here. What is the harm
|
|
# in having them and what do they remove. Do they interact
|
|
# adversely with the alien build support discussed above.
|
|
rmdir "$ROOTFS/usr/lib32" 2>/dev/null
|
|
rm -f "$ROOTFS/lib32" "$ROOTFS/lib64" "$ROOTFS/usr/lib64"
|
|
;;
|
|
esac
|
|
|
|
# Once base-files is configured and functional its possible to
|
|
# configure the rest of the system.
|
|
run_cmd_chroot "$ROOTFS" "xbps-reconfigure -a"
|
|
|
|
# At this point we're done running things in the chroot and we can
|
|
# clean up the shims. Failure to do this can result in things hanging
|
|
# when we try to delete the tmpdir.
|
|
cleanup_chroot
|
|
|
|
# Set the default password. Previous versions of this script used a
|
|
# chroot to do this, but that is unnecessary since chpasswd
|
|
# understands how to operate on chroots without actually needing to be
|
|
# chrooted. We also remove the lock file in this step to clean up the
|
|
# lock on the passwd database, lest it be left in the system and
|
|
# propogated to other points.
|
|
echo root:voidlinux | chpasswd -c SHA512 --root "$ROOTFS" || die "Could not set default credentials"
|
|
rm -f "$ROOTFS/etc/.pwd.lock"
|
|
|
|
# The cache isn't that useful since by the time the ROOTFS will be
|
|
# used it is likely to be out of date. Rather than shipping it around
|
|
# only for it to be out of date, we remove it now.
|
|
rm -rf "$ROOTFS/var/cache/*" 2>/dev/null
|
|
|
|
# Finally we can compress the tarball, the name will include the
|
|
# architecture and the date on which the tarball was built.
|
|
tarball=void-${XBPS_TARGET_ARCH}-ROOTFS-$(date '+%Y%m%d').tar.xz
|
|
run_cmd "tar -cp --posix --xattrs -C $ROOTFS . | xz -T0 -9 > $tarball "
|
|
|
|
# Now that we have the tarball we don't need the rootfs anymore, so we
|
|
# can get rid of it.
|
|
rm -rf "$ROOTFS"
|
|
|
|
# Last thing to do before closing out is to let the user know that
|
|
# this succeeded. This also ensures that there's something visible
|
|
# that the user can look for at the end of the script, which can make
|
|
# it easier to see what's going on if something above failed.
|
|
info_msg "Successfully created $tarball ($XBPS_TARGET_ARCH)"
|