User Tools

Site Tools


doc:howto:tce

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision
Previous revision
Next revision Both sides next revision
doc:howto:tce [2019/09/18 19:29]
stefanbaur [Configuring the Build] updated configuration for experimental ARM support
doc:howto:tce [2019/11/14 00:37]
stefanbaur [Configuring the Build]
Line 44: Line 44:
 # we have a successful connect on port 3142, assume # we have a successful connect on port 3142, assume
 # it's apt-cacher-ng and use it # it's apt-cacher-ng and use it
-#+
 if nc -z 127.0.0.1 3142 ; then  if nc -z 127.0.0.1 3142 ; then 
     # bad idea with apt-cacher-ng, but will work with e.g. squid     # bad idea with apt-cacher-ng, but will work with e.g. squid
Line 97: Line 97:
  
 # set boot loader type - leave this unchanged unless you really know what you're doing # set boot loader type - leave this unchanged unless you really know what you're doing
-if echo $LBX2GO_ARCH | awk '{print $2}' | grep -q "arm" ; then+if echo "$LBX2GO_ARCH| awk '{print $2}' | grep -q "arm" ; then
     # This is part of our experimental ARM support     # This is part of our experimental ARM support
     LBX2GO_BOOTLOADERPARAMNAME=" "     LBX2GO_BOOTLOADERPARAMNAME=" "
Line 138: Line 138:
  
 # This is part of our experimental ARM support # This is part of our experimental ARM support
-if echo "$LB_X2GO_ARCH" | grep -q 'arm'; then+if echo "$LBX2GO_ARCH" | grep -q 'arm'; then
     export LBX2GO_DEFAULTS+=" --bootstrap-qemu-arch arm64 \     export LBX2GO_DEFAULTS+=" --bootstrap-qemu-arch arm64 \
                               --bootstrap-qemu-static /usr/bin/qemu-arm-static"                               --bootstrap-qemu-static /usr/bin/qemu-arm-static"
Line 208: Line 208:
  
 # Set Directory name # Set Directory name
-LBX2GO_TCEDIR="./live-build-x2go-${LBX2GO_TIMESTAMP}-${LBX2GO_IMAGETYPE}-$(echo $LBX2GO_ARCH | awk '{print $2}')-${LBX2GO_CONFIG##*/}"+LBX2GO_TCEDIR="./live-build-x2go-${LBX2GO_TIMESTAMP}-${LBX2GO_IMAGETYPE}-$(echo "$LBX2GO_ARCH| awk '{print $2}')-${LBX2GO_CONFIG##*/}"
  
 if [ -z "$LBX2GO_ARCH" ] || if [ -z "$LBX2GO_ARCH" ] ||
Line 257: Line 257:
     if [ -f config/package-lists/firefox-langpacks.list.chroot ]; then     if [ -f config/package-lists/firefox-langpacks.list.chroot ]; then
             if [ -n "$LBX2GO_LANG" ]; then             if [ -n "$LBX2GO_LANG" ]; then
-                    for LBX2GO_SINGLE_LANG in $(echo $LBX2GO_LANG | tr ';' ' '); do+                    for LBX2GO_SINGLE_LANG in $(echo "$LBX2GO_LANG| tr ';' ' '); do
                             echo "LANG: '$LBX2GO_SINGLE_LANG'"                             echo "LANG: '$LBX2GO_SINGLE_LANG'"
                             sed -i -e 's/#firefox-esr-l10n-'$LBX2GO_SINGLE_LANG'$/firefox-esr-l10n-'$LBX2GO_SINGLE_LANG'/' config/package-lists/firefox-langpacks.list.chroot                             sed -i -e 's/#firefox-esr-l10n-'$LBX2GO_SINGLE_LANG'$/firefox-esr-l10n-'$LBX2GO_SINGLE_LANG'/' config/package-lists/firefox-langpacks.list.chroot
Line 275: Line 275:
         chmod 755 ./config/hooks/0112-remove-folders.hook.chroot         chmod 755 ./config/hooks/0112-remove-folders.hook.chroot
     fi     fi
 +
 +    if [ -n "$LB_APT_HTTP_PROXY" ] || [ -n "$LB_APT_FTP_PROXY" ]; then
 +        echo "NOTICE: apt proxy variable(s) is/are set."
 +        echo "NOTICE: Trying to use the proxy for all downloads."
 +        echo "NOTICE: If this fails, look for #SETPROXY in the $0 source."
 +        # Here, we should have reached a point where it is safe to point all proxy variables
 +        # at the apt-cacher-ng proxy.  If you're seeing errors during your build that hint
 +        # at files not being downloaded, disable these three entries.
 +        export https_proxy=$LB_APT_HTTP_PROXY
 + export http_proxy=$LB_APT_HTTP_PROXY
 +        export ftp_proxy=$LB_APT_FTP_PROXY
 +    fi
 +
 +    # This is a crude hack to detect crossbuilds for ARM on Intel/AMD hardware.
 +    # It makes some necessary changes, and also tries to speed up squashfs creation.
 +    if (uname -r | grep -q 'i.86' || uname -r | grep -q 'amd64') && \
 +       echo "$LB_X2GO_ARCH" | grep -q 'arm'; then
 +
 +        # This command removes all references to fuseext and x2gothinclient from the
 +        # package list files.  Currently needed as there are no ARM packages for either.
 +        echo "WARNING: Removing all references to fuseext and x2gothinclient from the build."
 +        sed -e 's/^.*fuseext.*$//g' -e 's/^.*x2gothinclient.*$//g' -i ./config/package-lists/*
 +
 + # This command removes the X2Go repository from the directory where additional
 +        # archives are stored.  Currently needed as the X2Go repository offers no arm64 
 +        # packages, but Debian Buster does - so that's what we're falling back to.
 +        echo "WARNING: Removing all references to the X2Go repository from the build."
 +        rm ./config/archives/*x2go*
 +
 + # The following is a hack to reduce squashfs creation time. We're replacing mksquashfs
 +        # in the changeroot environment with a wrapper script that drops the original 
 +        # mksquashfs call into a file.
 +
 +        # We need to do this as a background task, waiting for the mksquashfs executable to
 +        # appear in the changeroot; as the changeroot will only be created later on, once
 +        # lb build is called.
 +                
 +        # The other background task waits until the command file has been created, then
 +        # it applies some necessary patches to it, and starts the mksquashfs command natively
 +        # on the build host, rather than in the changeroot environment.
 +        # This is because in the changeroot, we'd be running the ARM mksquashfs in a qemu
 +        # software emulation of the ARM architecture, while on the host, we can use all the
 +        # native, raw CPU power and cores available to us.
 +
 +        # To make sure we don't have any lingering processes in the background, we're passing
 +        # our own PID along to the background tasks, and tell them to terminate if our PID
 +        # disappears while they're still in their waiting/looping state.
 +
 +        MASTERPID=$$
 +
 +        # Replace mksquashfs in chroot with script
 +        # (script will undo this upon completion)
 +        (
 +            # wait until the chroot has been populated or until our parent process dies
 +            while ! [ -x ./chroot/usr/bin/mksquashfs ]; do
 +                ps $MASTERPID >/dev/null || exit 1
 +                sleep 1
 +            done
 +            # make sure we don't overwrite the real executable if it has already been
 +            # moved out of the way
 +            if ! [ -x ./chroot/usr/bin/mksquashfs.real ]; then
 +                cp ./chroot/usr/bin/mksquashfs ./chroot/usr/bin/mksquashfs.real
 +            fi
 +           echo '#!/bin/bash' >./chroot/usr/bin/mksquashfs
 +           # log the name we've been called with and all parameters into this file
 +           echo 'echo "$0 $@" >/tmp/filesystem.squashfs.temp' >>./chroot/usr/bin/mksquashfs
 +           # once the native mksquashfs is complete, we will remove this file
 +           echo 'while [ -f /tmp/filesystem.squashfs.temp ]; do' >>./chroot/usr/bin/mksquashfs
 +           echo '        sleep 1' >>./chroot/usr/bin/mksquashfs
 +           echo 'done' >>./chroot/usr/bin/mksquashfs
 +           # so let's wait until it has been removed before deleting ourselves ...
 +           echo 'rm /usr/bin/mksquashfs' >>./chroot/usr/bin/mksquashfs
 +           # ... and moving the real executable back into its place
 +           echo 'mv /usr/bin/mksquashfs.real /usr/bin/mksquashfs' >>./chroot/usr/bin/mksquashfs
 +           chmod 755 ./chroot/usr/bin/mksquashfs
 +        ) &
 +
 +        # start the native mksquashfs after patching the parameters
 +        (
 +            # wait until the trigger file has been created or until our parent process dies
 +            while ! [ -f ./chroot/tmp/filesystem.squashfs.temp ]; do
 +                ps $MASTERPID >/dev/null || exit 1
 +                sleep 1
 +            done
 +            # using any of the available filters (x86, arm, armthumb) for the 
 +            # -Xbcj command results in an unusable squashfs on arm, so we drop the 
 +            # parameter completely if it's there.
 +            # also, all absolute paths (detected by beginning with " /") need to be
 +            # prefixed with "./chroot" so the mksquashfs outside the chroot knows where
 +            # to look for the corresponding paths/files.
 +            sed -e 's/ -Xbcj x86/ /g' -e 's# /# ./chroot/#g' -i \
 +                ./chroot/tmp/filesystem.squashfs.temp
 +            # now let's make this executable
 +            chmod 755 ./chroot/tmp/filesystem.squashfs.temp
 +
 +            # we also need to add some more excludes because they shouldn't end up
 +            # in the squashfs - no idea why we don't need them while inside the chroot ...
 +            echo 'proc/*' >>./chroot/excludes
 +            echo 'sys/*' >>./chroot/excludes
 +            echo 'dev/pts/*' >>/.chroot.excludes
 +            # now let's execute the script and, if it terminates without an error,
 +            # we'll move the newly created squashfs into the chroot where the chrooted
 +            # mksquashfs command would have created it; if that worked as well, we'll
 +            # remove the script file so our dummy mksquashfs inside the chroot knows
 +            # it's time to terminate itself.
 +            ./chroot/tmp/filesystem.squashfs.temp && \
 +            mv ./filesystem.squashfs ./chroot/ && \
 +            rm ./chroot/tmp/filesystem.squashfs.temp
 +        ) &
 +    fi
 +
     if lb build ; then     if lb build ; then
         echo -e "Build is done: '$LBX2GO_TCEDIR'"         echo -e "Build is done: '$LBX2GO_TCEDIR'"
Line 282: Line 393:
  
         if [ "$LBX2GO_IMAGETYPE" = "hdd" ] ; then         if [ "$LBX2GO_IMAGETYPE" = "hdd" ] ; then
-                ln ./live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').img  +                ln ./live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').img \ 
-                   ./x2go-tce-live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').img+                   ./x2go-tce-live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').img
         fi         fi
         if [ "$LBX2GO_IMAGETYPE" = "netboot" ] ; then         if [ "$LBX2GO_IMAGETYPE" = "netboot" ] ; then
Line 296: Line 407:
         if [ "$LBX2GO_IMAGETYPE" = "iso" ] || [ "$LBX2GO_IMAGETYPE" = "iso-hybrid" ] ; then         if [ "$LBX2GO_IMAGETYPE" = "iso" ] || [ "$LBX2GO_IMAGETYPE" = "iso-hybrid" ] ; then
             genisoimage -o ./x2go-tce-squashfs-only.iso -R -J -graft-points live/filesystem.squashfs=./x2go-tce-filesystem.squashfs             genisoimage -o ./x2go-tce-squashfs-only.iso -R -J -graft-points live/filesystem.squashfs=./x2go-tce-filesystem.squashfs
-            if [ -e ./live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').hybrid.iso ] ; then +            if [ -e ./live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').hybrid.iso ] ; then 
-                    ln ./live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').hybrid.iso \ +                    ln ./live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').hybrid.iso \ 
-                       ./original-x2go-tce-live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').hybrid.iso +                       ./original-x2go-tce-live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').hybrid.iso 
-            elif [ -e ./live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').iso ] ; then +            elif [ -e ./live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').iso ] ; then 
-                    ln ./live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').iso \ +                    ln ./live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').iso \ 
-                       ./original-x2go-tce-live-image-$(echo $LBX2GO_ARCH | awk '{print $2}').iso+                       ./original-x2go-tce-live-image-$(echo "$LBX2GO_ARCH| awk '{print $2}').iso
             fi             fi
             mv ./x2go-tce-filesystem.squashfs ./original-x2go-tce-filesystem.squashfs             mv ./x2go-tce-filesystem.squashfs ./original-x2go-tce-filesystem.squashfs
doc/howto/tce.txt · Last modified: 2024/01/26 19:49 by stefanbaur