From a4a4074948874ea954dcb62c2d018234f5f71f9f Mon Sep 17 00:00:00 2001 From: mappu Date: Tue, 29 Apr 2025 22:48:09 +1200 Subject: [PATCH] android: major rework for all-in-one android support --- cmd/android-mktemplate/android-mktemplate.sh | 66 --- cmd/android-stub-gen/android-stub-gen.sh | 106 ----- cmd/miqt-docker/android-build.sh | 415 ++++++++++++++++++ cmd/miqt-docker/main.go | 4 +- cmd/miqt-docker/tasks.go | 22 + ...id-armv8a-go1.23-qt5.15-dynamic.Dockerfile | 7 +- ...oid-armv8a-go1.23-qt6.6-dynamic.Dockerfile | 3 - 7 files changed, 444 insertions(+), 179 deletions(-) delete mode 100755 cmd/android-mktemplate/android-mktemplate.sh delete mode 100755 cmd/android-stub-gen/android-stub-gen.sh create mode 100644 cmd/miqt-docker/android-build.sh diff --git a/cmd/android-mktemplate/android-mktemplate.sh b/cmd/android-mktemplate/android-mktemplate.sh deleted file mode 100755 index 8ccc3f0a..00000000 --- a/cmd/android-mktemplate/android-mktemplate.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# -# android-mktemplate generates a template json file suitable for use with the -# androiddeployqt tool. - -set -eu - -# QT_PATH is already pre-set in our docker container environment. Includes trailing slash. -QT_PATH=${QT_PATH:-/usr/local/Qt-5.15.13/} -QT_ANDROID=${QT_ANDROID:-$QT_PATH} - -ndk_version() { - ls /opt/android-sdk/ndk/ | tail -n1 -} - -target_sdk_version() { - ls /opt/android-sdk/platforms | tail -n1 | sed -re 's/android-//' -} - -build_tools_version() { - ls /opt/android-sdk/build-tools | tail -n1 -} - -extra_libs() { - if [[ -d /opt/android_openssl ]] ; then - # Our miqt Qt5 container includes these extra .so libraries - # However, the aqtinstall-based Qt 6 container does not use them - echo "/opt/android_openssl/ssl_1.1/arm64-v8a/libssl_1_1.so,/opt/android_openssl/ssl_1.1/arm64-v8a/libcrypto_1_1.so" - fi -} - -main() { - - if [[ $# -ne 2 ]] ; then - echo "Usage: android-mktemplate.sh appname output.json" >&2 - exit 1 - fi - local ARG_APPNAME="$1" - local ARG_DESTFILE="$2" - - # Available fields are documented in the template file at - # @ref /usr/local/Qt-5.15.13/mkspecs/features/android/android_deployment_settings.prf - cat > "${ARG_DESTFILE}" <&2 - exit 1 - fi - local ARG_SOURCE_SOFILE="$1" - local ARG_FUNCTIONNAME="$2" - local ARG_DEST_SOFILE="$3" - local ARG_QTVERSION="${4:---qt5}" - - local tmpdir=$(mktemp -d) - trap "rm -r ${tmpdir}" EXIT - - echo "- Using temporary directory: ${tmpdir}" - - echo "Generating stub..." - - cat > $tmpdir/miqtstub.cpp < -#include -#include - -typedef void goMainFunc_t(); - -int main(int argc, char** argv) { - __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "Starting up"); - - void* handle = dlopen("$(basename "$ARG_SOURCE_SOFILE")", RTLD_LAZY); - if (handle == NULL) { - __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: null handle opening so: %s", dlerror()); - exit(1); - } - - void* goMain = dlsym(handle, "${ARG_FUNCTIONNAME}"); - if (goMain == NULL) { - __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: null handle looking for function: %s", dlerror()); - exit(1); - } - - __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: Found target, calling"); - - // Cast to function pointer and call - goMainFunc_t* f = (goMainFunc_t*)goMain; - f(); - - __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: Target function returned"); - return 0; -} - -EOF - - # Compile - # Link with Qt libraries so that androiddeployqt detects us as being the - # main shared library - - if [[ $ARG_QTVERSION == '--qt5' ]] ; then - - # QT_PATH is already pre-set in our docker container environment. Includes trailing slash. - QT_PATH=${QT_PATH:-/usr/local/Qt-5.15.13/} - echo "- Found Qt path: ${QT_PATH}" - - $CXX -shared \ - -ldl \ - -llog \ - -L${QT_PATH}plugins/platforms -lplugins_platforms_qtforandroid_arm64-v8a \ - $(pkg-config --libs Qt5Widgets) \ - $(pkg-config --libs Qt5AndroidExtras) \ - $tmpdir/miqtstub.cpp \ - "-Wl,-soname,$(basename "$ARG_DEST_SOFILE")" \ - -o "$ARG_DEST_SOFILE" - - elif [[ $ARG_QTVERSION == '--qt6' ]] ; then - - # QT_ANDROID is already pre-set in our docker container environment. Does NOT include trailing slash - QT_ANDROID=${QT_ANDROID:-/opt/Qt/6.6.1/android_arm64_v8a} - echo "- Found Qt path: ${QT_ANDROID}" - - # There is no AndroidExtras in Qt 6 - - $CXX -shared \ - -ldl \ - -llog \ - -L${QT_ANDROID}/plugins/platforms -lplugins_platforms_qtforandroid_arm64-v8a \ - $(pkg-config --libs Qt6Widgets) \ - $tmpdir/miqtstub.cpp \ - "-Wl,-soname,$(basename "$ARG_DEST_SOFILE")" \ - -o "$ARG_DEST_SOFILE" - - else - echo "Unknown Qt version argument "${ARG_QTVERSION}" (expected --qt5 or --qt6)" >&2 - exit 1 - fi - - echo "Done." -} - -main "$@" diff --git a/cmd/miqt-docker/android-build.sh b/cmd/miqt-docker/android-build.sh new file mode 100644 index 00000000..7d613f50 --- /dev/null +++ b/cmd/miqt-docker/android-build.sh @@ -0,0 +1,415 @@ +#!/bin/bash +# +# android-build.sh allows building a MIQT Go application for Android. +# For details, see the top-level README.md file. + +set -Eeuo pipefail + +# QT_PATH is pre-set in the Qt 5 docker container environment. Includes trailing slash +# QT_ANDROID is pre-set in the Qt 6 docker container environment +QT_PATH=${QT_PATH:-/usr/local/Qt-5.15.13/} +QT_ANDROID=${QT_ANDROID:-$QT_PATH} + +export LC_ALL=C.UTF-8 + +# get_app_name returns the android app's name. This affects the default name +# in deployment-settings.json and the generated lib.so names. +# You can still customise the package name and the package ID in the xml +# files after generation. +get_app_name() { + basename "$(pwd)" +} + +get_stub_soname() { + # libRealAppName_arm64-v8a.so + echo "lib$(get_app_name)_arm64-v8a.so" +} + +get_go_soname() { + echo "libMiqtGolangApp_arm64-v8a.so" +} + +ndk_version() { + ls /opt/android-sdk/ndk/ | tail -n1 +} + +target_sdk_version() { + ls /opt/android-sdk/platforms | tail -n1 | sed -re 's/android-//' +} + +build_tools_version() { + ls /opt/android-sdk/build-tools | tail -n1 +} + +# extra_libs returns a comma-separated list of extra libraries to include in +# the apk package +extra_libs() { + if [[ -d /opt/android_openssl ]] ; then + # Our miqt Qt5 container includes these extra .so libraries + # However, the aqtinstall-based Qt 6 container does not use them + echo "/opt/android_openssl/ssl_1.1/arm64-v8a/libssl_1_1.so,/opt/android_openssl/ssl_1.1/arm64-v8a/libcrypto_1_1.so" + fi +} + +# generate_template_contents produces a deployment settings JSON file that is +# understood by the androiddeployqt program. +# Available fields are documented in the template file at: +# @ref /usr/local/Qt-5.15.13/mkspecs/features/android/android_deployment_settings.prf +generate_template_contents() { + + cat < $STUBNAME < +#include +#include + +typedef void goMainFunc_t(); + +int main(int argc, char** argv) { + __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "Starting up"); + + void* handle = dlopen("$(get_go_soname)", RTLD_LAZY); + if (handle == NULL) { + __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: null handle opening so: %s", dlerror()); + exit(1); + } + + void* goMain = dlsym(handle, "AndroidMain"); + if (goMain == NULL) { + __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: null handle looking for function: %s", dlerror()); + exit(1); + } + + __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: Found target, calling"); + + // Cast to function pointer and call + goMainFunc_t* f = (goMainFunc_t*)goMain; + f(); + + __android_log_print(ANDROID_LOG_VERBOSE, "miqt_stub", "miqt_stub: Target function returned"); + return 0; +} + +EOF + + # Compile + # Link with Qt libraries so that androiddeployqt detects us as being the + # main shared library + + $CXX -shared \ + -ldl \ + -llog \ + -L${QT_ANDROID}/plugins/platforms -lplugins_platforms_qtforandroid_arm64-v8a \ + $(pkg-config --libs $(pkg_config_dependencies)) \ + ${STUBNAME} \ + "-Wl,-soname,$(basename "$(get_stub_soname)")" \ + -o "android-build/libs/arm64-v8a/$(get_stub_soname)" + + rm "${STUBNAME}" +} + +# require_is_main_package verifies that this is the main Go package. +require_is_main_package() { + if ! grep -Fq 'package main' *.go ; then + echo "This doesn't seem to be the main package" >&2 + exit 1 + fi +} + +# patch_app_main sets up the startup go files so that the go program can be +# built either as c-shared for Android or as a normal program for desktop OSes. +patch_app_main() { + + # Replace func main() with app_main() + + for srcfile in *.go ; do + sed -i -re 's/^func main\(\) \{/func app_main() {/' "$srcfile" + done + + # Add shim startup files + + cat < startup_android.go +//go:build android +// +build android + +package main + +import "C" // Required for export support + +//export AndroidMain +func AndroidMain() { + app_main() +} + +func main() { + // Must be empty +} + +EOF + + cat < startup_other.go +//go:build !android +// +build !android + +package main + +func main() { + app_main() +} + +EOF + + gofmt -w startup_android.go || true + gofmt -w startup_other.go || true + + # Done +} + +# unpatch_app_main undoes the transformation from patch_app_main. +unpatch_app_main() { + + # Replace func main() with app_main() + + for srcfile in *.go ; do + sed -i -re 's/^func app_main\(\) \{/func main() {/' "$srcfile" + done + + # Remove extra startup files + + rm startup_android.go || true + rm startup_other.go || true +} + +# build_app_so compiles the Go app as a c-shared .so. +build_app_so() { + go build \ + -buildmode c-shared \ + -ldflags "-s -w -extldflags -Wl,-soname,$(get_go_soname)" \ + -o "android-build/libs/arm64-v8a/$(get_go_soname)" +} + +sdkmanager() { + echo /opt/android-sdk/cmdline-tools/*/bin/sdkmanager +} + +# build_apk calls androiddeployqt to package the android-build directory into +# the final apk. +build_apk() { + + # Qt 6 androiddeployqt: Understands the QT_ANDROID_KEYSTORE_STORE_PASS in env + # Qt 5 androiddeployqt: Doesn't - any use of `--sign keystore alias` here + # requires stdin prompt but doesn't pass androiddeployqt's stdin through + # to jarsigner subprocess + # Either way, don't sign the app here, rely on separate jarsigner command + + # Work around an issue with Qt 6 sporadically failing to detect that + # we have a valid Qt platform plugin + # TODO why does this happen? Is it related to file sort ordering? + # It really does fix itself after multiple attempts (usually less than 5) + # - When it fails: Error: qmlimportscanner not found at /qmlimportscanner + # - When it works: Error: qmlimportscanner not found at libexec/qmlimportscanner + while ! androiddeployqt \ + --input ./deployment-settings.json \ + --output ./android-build/ \ + --release \ + 2> >(tee /tmp/androiddeployqt.stderr.log >&2) ; do + + if grep -Fq 'Make sure the app links to Qt Gui library' /tmp/androiddeployqt.stderr.log ; then + echo "Detected temporary problem with Qt plugin detection. Retrying..." + sleep 1 + + else + # real error + exit 1 + fi + done + + local OUTAPK=$(get_app_name).apk + rm "$OUTAPK" || true + + # Zipalign + echo "Zipalign..." + /opt/android-sdk/build-tools/*/zipalign \ + -p 4 \ + ./android-build/build/outputs/apk/release/android-build-release-unsigned.apk \ + "$OUTAPK" + + # Sign + echo "Signing..." + #jarsigner \ + # -verbose \ + # -sigalg SHA256withRSA -digestalg SHA256 \ + # -keystore ./android.keystore \ + # "$OUTAPK" \ + # "${QT_ANDROID_KEYSTORE_ALIAS}" \ + # -storepass:env QT_ANDROID_KEYSTORE_STORE_PASS \ + # -keypass:env QT_ANDROID_KEYSTORE_KEY_PASS + + /opt/android-sdk/build-tools/*/apksigner \ + sign \ + --ks ./android.keystore \ + --ks-key-alias "${QT_ANDROID_KEYSTORE_ALIAS}" \ + --ks-pass env:QT_ANDROID_KEYSTORE_STORE_PASS \ + --key-pass env:QT_ANDROID_KEYSTORE_KEY_PASS \ + "$OUTAPK" +} + +# detect_env_qt_version detects the system's current Qt version. +detect_env_qt_version() { + if qmake --version | fgrep -q 'Qt version 5' ; then + echo "qt5" + return 0 + + elif qmake --version | fgrep -q 'Qt version 6' ; then + echo "qt6" + return 0 + + else + echo "Missing Qt tools in PATH" >&2 + exit 1 + fi +} + +# detect_miqt_qt_version echoes either "qt5", "qt6", or exits bash. +detect_miqt_qt_version() { + local IS_QT5=false + if grep -qF '"github.com/mappu/miqt/qt"' *.go ; then + IS_QT5=true + fi + + local IS_QT6=false + if grep -qF '"github.com/mappu/miqt/qt6"' *.go ; then + IS_QT6=true + fi + + if [[ $IS_QT5 == true && $IS_QT6 == true ]] ; then + echo "Found qt5 and qt6 imports, confused about what to do next" >&2 + exit 1 + + elif [[ $IS_QT5 == true ]] ; then + echo "qt5" + return 0 + + elif [[ $IS_QT6 == true ]] ; then + echo "qt6" + return 0 + + else + echo "Found neither qt5 nor qt6 imports. Is this a MIQT Qt app?" >&2 + exit 1 + fi +} + +generate_default_keystore() { + + local GENPASS=storepass_$(cat /dev/urandom | head -c64 | md5sum | cut -d' ' -f1) + + keytool \ + -genkeypair \ + -dname "cn=Miqt, ou=Miqt, o=Miqt, c=US" \ + -keyalg RSA \ + -alias miqt \ + -keypass "${GENPASS}" \ + -keystore ./android.keystore \ + -storepass "${GENPASS}" \ + -validity 20000 + + echo "QT_ANDROID_KEYSTORE_PATH=./android.keystore" > android.keystore.env + echo "QT_ANDROID_KEYSTORE_ALIAS=miqt" >> android.keystore.env + echo "QT_ANDROID_KEYSTORE_STORE_PASS=${GENPASS}" >> android.keystore.env + echo "QT_ANDROID_KEYSTORE_KEY_PASS=${GENPASS}" >> android.keystore.env +} + +# main is the entrypoint for android-build.sh. +main() { + + if [[ $(detect_env_qt_version) != $(detect_miqt_qt_version) ]] ; then + echo "The system is $(detect_env_qt_version) but the app uses $(detect_miqt_qt_version). Is this the right container?" >&2 + exit 1 + fi + + require_is_main_package + + # Rebuild deployment-settings.json + if [[ ! -f deployment-settings.json ]] ; then + echo "Generating deployment-settings.json..." + generate_template_contents > deployment-settings.json + fi + + mkdir -p android-build/libs/arm64-v8a + + if [[ ! -f android-build/libs/arm64-v8a/$(get_stub_soname) ]] ; then + echo "Generating stub so..." + android_stub_gen + fi + + # Rebuild miqt_golang_app.so + echo "Compiling Go app..." + patch_app_main + build_app_so + unpatch_app_main + + # Keypair + if [[ ! -f android.keystore || ! -f android.keystore.env ]] ; then + echo "Signing keystore not found, generating a default one..." + generate_default_keystore + fi + + # Load keypair credentials into exported env vars + set -o allexport + source android.keystore.env + set +o allexport + + # Generate .apk + echo "Packaging APK..." + build_apk + + echo "Complete" +} + +main "$@" diff --git a/cmd/miqt-docker/main.go b/cmd/miqt-docker/main.go index 044d56f0..ea367fce 100644 --- a/cmd/miqt-docker/main.go +++ b/cmd/miqt-docker/main.go @@ -213,7 +213,9 @@ func getDockerRunArgsForGlob(dockerfiles []fs.DirEntry, containerNameGlob string return nil, err } - fullCommand = append(fullCommand, `-v`, basedir+`:/src`, `-w`, filepath.Join(`/src`, relCwd)) + mountDir := `/src/` + filepath.Base(cwd) // Don't use /src directly, otherwise -android-build will not know the package name for top-level builds + + fullCommand = append(fullCommand, `-v`, basedir+`:`+mountDir, `-w`, filepath.Join(mountDir, relCwd)) fullCommand = append(fullCommand, `-e`, `HOME=/tmp`) diff --git a/cmd/miqt-docker/tasks.go b/cmd/miqt-docker/tasks.go index 260bba4b..903f3668 100644 --- a/cmd/miqt-docker/tasks.go +++ b/cmd/miqt-docker/tasks.go @@ -1,11 +1,18 @@ package main import ( + "bytes" + "embed" "errors" "fmt" "os/exec" ) +//go:embed android-build.sh +var embedAndroidBuildSh []byte + +var _ embed.FS // Workaround to allow import of package `embed` + // evaluateTask turns the supplied process arguments into real arguments to // execute, handling quick command recipes as well as arbitrary execution func evaluateTask(taskArgs []string) (retArgs []string, fixup func(*exec.Cmd), allowTty bool, err error) { @@ -26,6 +33,14 @@ func evaluateTask(taskArgs []string) (retArgs []string, fixup func(*exec.Cmd), a // + stdinFrom := func(stdinBytes []byte) func(*exec.Cmd) { + return func(c *exec.Cmd) { + c.Stdin = bytes.NewReader(stdinBytes) + } + } + + // + if taskArgs[0][0] != '-' { // Task does not start with a hyphen = plain command retArgs = taskArgs @@ -49,6 +64,13 @@ func evaluateTask(taskArgs []string) (retArgs []string, fixup func(*exec.Cmd), a retArgs = append(retArgs, taskArgs[1:]...) return + case `-android-build`: + retArgs = []string{"/bin/bash", "-s"} + retArgs = append(retArgs, taskArgs[1:]...) + fixup = stdinFrom(embedAndroidBuildSh) + allowTty = false + return + default: return nil, nil, false, fmt.Errorf("Unrecognized task %q", taskArgs[0]) } diff --git a/docker/android-armv8a-go1.23-qt5.15-dynamic.Dockerfile b/docker/android-armv8a-go1.23-qt5.15-dynamic.Dockerfile index 69424071..e63de6c9 100644 --- a/docker/android-armv8a-go1.23-qt5.15-dynamic.Dockerfile +++ b/docker/android-armv8a-go1.23-qt5.15-dynamic.Dockerfile @@ -4,9 +4,6 @@ RUN wget 'https://go.dev/dl/go1.23.1.linux-amd64.tar.gz' && \ tar x -C /usr/local/ -f go1.23.1.linux-amd64.tar.gz && \ rm go1.23.1.linux-amd64.tar.gz -COPY cmd/android-stub-gen/android-stub-gen.sh /usr/local/bin/android-stub-gen.sh -COPY cmd/android-mktemplate/android-mktemplate.sh /usr/local/bin/android-mktemplate.sh - ENV PATH=/usr/local/go/bin:/opt/cmake/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/Qt-5.15.13/bin:/opt/android-sdk/cmdline-tools/tools/bin:/opt/android-sdk/tools:/opt/android-sdk/tools/bin:/opt/android-sdk/platform-tools # The pkg-config definitions were all installed with platform-specific suffixes @@ -14,6 +11,10 @@ ENV PATH=/usr/local/go/bin:/opt/cmake/bin:/usr/local/sbin:/usr/local/bin:/usr/sb # This container is targeting armv8-a, so set up simple symlinks RUN /bin/bash -c 'cd /usr/local/Qt-5.15.13/lib/pkgconfig ; for f in *_arm64-v8a.pc ; do cp $f "$(basename -s _arm64-v8a.pc "$f").pc"; done' +# This is gross but (A) it's containerized and (B) allows --uid=1000 to perform builds +# Only needed for certain problematic versions of the Android SDK; a readonly SDK works in both older+newer SDKs +RUN /bin/bash -c 'find /opt/android-sdk/ -type d -exec chmod 777 {} \; && find /opt/android-sdk/ -perm 660 -exec chmod 666 {} \;' + ENV CC=/opt/android-sdk/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang ENV CXX=/opt/android-sdk/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang++ ENV CGO_ENABLED=1 diff --git a/docker/android-armv8a-go1.23-qt6.6-dynamic.Dockerfile b/docker/android-armv8a-go1.23-qt6.6-dynamic.Dockerfile index deb2f147..249718ad 100644 --- a/docker/android-armv8a-go1.23-qt6.6-dynamic.Dockerfile +++ b/docker/android-armv8a-go1.23-qt6.6-dynamic.Dockerfile @@ -9,9 +9,6 @@ RUN wget 'https://go.dev/dl/go1.23.6.linux-amd64.tar.gz' && \ tar x -C /usr/local/ -f go1.23.6.linux-amd64.tar.gz && \ rm go1.23.6.linux-amd64.tar.gz -COPY cmd/android-stub-gen/android-stub-gen.sh /usr/local/bin/android-stub-gen.sh -COPY cmd/android-mktemplate/android-mktemplate.sh /usr/local/bin/android-mktemplate.sh - # Fix up pkg-config definitions: # 1. There are only pkg-config definitions included for gcc_64 (Linux native), not for the android_arm64_v8a target we actually want # 2. It looks for `Libs: -L${libdir} -lQt6Widgets` but the file is named libQt6Widgets_arm64-v8a.so