From 10568660ac2398bdf871211c0b68114db46421c9 Mon Sep 17 00:00:00 2001 From: Michael Aldridge Date: Wed, 9 Aug 2017 20:58:32 -0700 Subject: [PATCH] Clean up chroot/QEMU logic The logic that sets up the QEMU static binary to permit running alien binary formats is quite complex. This commit cleans up a lot of the logic around this and adds a huge number of comments to try and clarify why things happen when they do. Ultimately this commit provides a lot of the ground work to factor out all of the chroot setup into a seperate shell file that can be sourced by other mklive tasks that need to operate inside of a chroot with a possibly alien executable format. --- mkrootfs.sh.in | 176 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 48 deletions(-) diff --git a/mkrootfs.sh.in b/mkrootfs.sh.in index 5ca4020..261ec22 100644 --- a/mkrootfs.sh.in +++ b/mkrootfs.sh.in @@ -69,14 +69,30 @@ _EOF } mount_pseudofs() { + # This function ensures that the psuedofs mountpoints are present + # in the chroot. Strictly they are not necessary to have for many + # commands, but bind-mounts are cheap and it isn't too bad to just + # mount them all the time. for f in dev proc sys; do + # In a naked chroot there is nothing to bind the mounts to, so + # we need to create directories for these first. [ ! -d "$ROOTFS/$f" ] && mkdir -p "$ROOTFS/$f" - mount -r --bind /$f "$ROOTFS/$f" + if ! mountpoint -q "$ROOTFS/$f" ; then + # It is VERY important that this only happen if the + # pseudofs isn't already mounted. If it already is then + # this is virtually impossible to troubleshoot because it + # looks like the subsequent umount just isn't working. + mount -r --bind /$f "$ROOTFS/$f" + fi done } umount_pseudofs() { - umount -f /proc/sys/fs/binfmt_misc >/dev/null 2>&1 + # This function cleans up the mounts in the chroot. Failure to + # clean up these mounts will prevent the tmpdir from being + # deletable instead throwing the error "Device or Resource Busy". + # The '-f' option is passed to umount to account for the + # contingency where the psuedofs mounts are not present. if [ -d "${ROOTFS}" ]; then for f in dev proc sys; do umount -f "$ROOTFS/$f" >/dev/null 2>&1 @@ -107,25 +123,118 @@ run_cmd() { eval "$@" } +run_cmd_chroot() { + # General purpose chroot function which makes sure the chroot is + # prepared. This function takes 2 arguments, the location to + # chroot to and the command to run. + + # This is an idempotent function, it is safe to call every time + # before entering the chroot. This has the advantage of making + # execution in the chroot appear as though it "Just Works(tm)". + register_binfmt + + # Before we step into the chroot we need to make sure the + # pseudo-filesystems are ready to go. Not all commands will need + # this, but its still a good idea to call it here anyway. + mount_pseudofs + + # With assurance that things will run now we can jump into the + # chroot and run stuff! + chroot "$1" sh -c "$2" +} + +cleanup_chroot() { + # This function cleans up the chroot shims that are used by QEMU + # to allow builds on alien platforms. It takes no arguments but + # expects the global $ROOTFS variable to be set. + + # Un-Mount the pseudofs mounts if they were mounted + umount_pseudofs + + # If a QEMU binary was copied in, remove that as well + if [ -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then + rm "$ROOTFS/usr/bin/$QEMU_BIN" + fi +} + # TODO: Figure out how to register the binfmt for x86_64 and for i686 # to facilitate building on alien build systems. register_binfmt() { - mountpoint -q /proc/sys/fs/binfmt_misc || modprobe -q binfmt_misc; mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null - case "${QEMU_BIN}" in - qemu-arm-static) - echo ':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:' > /proc/sys/fs/binfmt_misc/register 2>/dev/null + # This function sets up everything that is needed to be able to + # chroot into a ROOTFS and be able to run commands there. This + # really matters on platforms where the host architecture is + # different from the target, and you wouldn't be able to run + # things like xbps-reconfigure -a. This function is idempotent + # (You can run it multiple times without modifying state). This + # function takes no arguments, but does expect the global variable + # $XBPS_TARGET_ARCH to be set. + + # This select sets up the "magic" bytes in /proc that let the + # kernel select an alternate interpreter. More values for this + # map can be obtained from here: + # https://github.com/qemu/qemu/blob/master/scripts/qemu-binfmt-conf.sh + case "${XBPS_TARGET_ARCH}" in + armv*) + _cpu=arm + _magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00" + _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff" + QEMU_BIN=qemu-arm-static ;; - qemu-aarch64-static) - echo ':arm64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/bin/qemu-aarch64-static:' > /proc/sys/fs/binfmt_misc/register 2>/dev/null + aarch64*) + _cpu=aarch64 + _magic="\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7" + _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff" + QEMU_BIN=qemu-aarch64-static ;; - qemu-mipsel-static) - echo ':mipsel:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-mipsel-static:' > /proc/sys/fs/binfmt_misc/register 2>/dev/null + mipsel*) + _cpu=mipsel + _magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00" + _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff" + QEMU_BIN=qemu-mipsel-static + ;; + *86*) + info_msg "FIXME: Assuming that x86 instructions are native" + QEMU_BIN=NATIVE ;; *) die "Unknown target architecture!" ;; esac - cp -f "$(which "$QEMU_BIN")" "$ROOTFS/usr/bin" || die "failed to copy $QEMU_BIN to the ROOTFS" + + # In the special case where the build is native we can return + # without doing anything else + if [ "$QEMU_BIN" = "NATIVE" ] ; then + return + fi + + # For builds that do not match the host architecture, the correct + # qemu binary will be required. + if ! $QEMU_BIN -version >/dev/null 2>&1; then + die "$QEMU_BIN binary is missing in your system, exiting." + fi + + # In order to use the binfmt system the binfmt_misc mountpoint + # must exist inside of proc + if ! mountpoint -q /proc/sys/fs/binfmt_misc ; then + modprobe -q binfmt_misc + mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null + fi + + # Only register if the map is incomplete + if [ ! -f /proc/sys/fs/binfmt_misc/qemu-$_cpu ] ; then + echo ":qemu-$_cpu:M::$_magic:$_mask:$QEMU_BIN:" > /proc/sys/fs/binfmt_misc/register 2>/dev/null + fi + + # If the static binary isn't in the chroot then the chroot will + # fail. The kernel knows about the map but without the static + # version there's no interpreter in the chroot, only the + # dynamically linked one in the host. To simplify things we just + # use the static one always and make sure it shows up at the same + # place in the host and the chroot. + if [ ! -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then + cp -f "$(which "$QEMU_BIN")" "$ROOTFS/usr/bin" || + die "Could not install $QEMU_BIN to $ROOTFS/usr/bin/" + fi } # @@ -156,18 +265,6 @@ if [ -z "$XBPS_TARGET_ARCH" ]; then usage; exit 1 fi -# This select maps the architectures to the appropriate QEMU binaries -# since this mapping isn't something that can just be subbed in for -# easily. -case "$XBPS_TARGET_ARCH" in - i686*) QEMU_BIN=qemu-i386-static ;; - x86_64*) QEMU_BIN=qemu-x86_64-static ;; - armv*) QEMU_BIN=qemu-arm-static ;; - aarch64*) QEMU_BIN=qemu-aarch64-static ;; - mipsel*) QEMU_BIN=qemu-mipsel-static ;; - *) die "Unknown target architecture" ;; -esac - # If the repository hasn't already been set, we set it to a sane value # here. These should all resolve even if they won't have the # appropriate repodata files for the selected architecture. @@ -186,12 +283,6 @@ for f in chroot tar xbps-install xbps-reconfigure xbps-query; do fi done -# For builds that do not match the host architecture, the correct qemu -# binary will also be required. -if ! $QEMU_BIN -version >/dev/null 2>&1; then - die "$QEMU_BIN binary is missing in your system, exiting." -fi - # We need to operate on a tempdir, if this fails to create, it is # absolutely crucial to bail out so that we don't hose the system that # is running the script. @@ -248,22 +339,16 @@ case "$XBPS_TARGET_ARCH" in run_cmd "XBPS_ARCH=${XBPS_TARGET_ARCH} xbps-reconfigure -r $ROOTFS base-files" ;; *) - # This case handles configuration of the system when it won't - # work directly with the host ELF infrastructure. Before - # continuing its necessary to determine the correct magic - # numbers and load them into the kernel so that it will defer - # to the appropriate interpreter as defined by $QEMU_BIN - register_binfmt - # This step sets up enough of the base-files that the chroot # will work and they can be reconfigured natively. Without # this step there isn't enough configured for ld to work. + # This step runs as the host architecture. run_cmd "xbps-reconfigure -r $ROOTFS base-files" - + # Now running as the target system, this step reconfigures the # base-files completely. Certain things just won't work in # the first pass, so this cleans up any issues that linger. - run_cmd "chroot $ROOTFS env -i xbps-reconfigure -f base-files" + run_cmd_chroot "$ROOTFS" "env -i xbps-reconfigure -f base-files" # TODO: determine why these lines are here. What is the harm # in having them and what do they remove. Do they interact @@ -275,11 +360,12 @@ esac # Once base-files is configured and functional its possible to # configure the rest of the system. -run_cmd "chroot $ROOTFS xbps-reconfigure -a" +run_cmd_chroot "$ROOTFS" "xbps-reconfigure -a" -# At this point we're done running things that needed to be done with -# the pseudo filesystems to be mounted, so we can clean that up. -umount_pseudofs +# At this point we're done running things in the chroot and we can +# clean up the shims. Failure to do this can result in things hanging +# when we try to delete the tmpdir. +cleanup_chroot # Set the default password. Previous versions of this script used a # chroot to do this, but that is unnecessary since chpasswd @@ -295,12 +381,6 @@ rm -f "$ROOTFS/etc/.pwd.lock" # only for it to be out of date, we remove it now. rm -rf "$ROOTFS/var/cache/*" 2>/dev/null -# If we needed to copy in a QEMU_BIN executable, that needs to be -# removed before packaging up the shiny new ROOTFS. This could be -# wrapped in a conditional, but its much easier to just remove the -# binary location on the off chance its there. -rm -f "$ROOTFS/usr/bin/$QEMU_BIN" - # Finally we can compress the tarball, the name will include the # architecture and the date on which the tarball was built. tarball=void-${XBPS_TARGET_ARCH}-ROOTFS-$(date '+%Y%m%d').tar.xz