diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c6bfb77..f6a09a68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,7 @@ on: branches: - main pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -49,3 +50,117 @@ jobs: -skipMacroValidation -skipPackagePluginValidation \ -scheme XKit -destination generic/platform=iOS \ | xcbeautify + build-android: + # Cross-compile for Android (bionic) from Linux with the Swift SDK, + # validating the platform guards used for native Android hosts. + runs-on: ubuntu-24.04 + env: + SWIFT_VERSION: 6.3.2 + # Keep in sync with the toolchain version above. + ANDROID_SDK_CHECKSUM: 939e933549d12d28f2e0bf71019d734d309859e9773c572657ce565a81f85d68 + NDK_VERSION: 27c + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Install Swift toolchain + run: | + curl -sfL "https://download.swift.org/swift-${SWIFT_VERSION}-release/ubuntu2404/swift-${SWIFT_VERSION}-RELEASE/swift-${SWIFT_VERSION}-RELEASE-ubuntu24.04.tar.gz" \ + | tar -xzf - -C "$HOME" + echo "$HOME/swift-${SWIFT_VERSION}-RELEASE-ubuntu24.04/usr/bin" >> "$GITHUB_PATH" + - name: Install Android NDK + run: | + curl -sfL -o ndk.zip "https://dl.google.com/android/repository/android-ndk-r${NDK_VERSION}-linux.zip" + unzip -q ndk.zip -d "$HOME" + echo "ANDROID_NDK_HOME=$HOME/android-ndk-r${NDK_VERSION}" >> "$GITHUB_ENV" + - name: Install Swift SDK for Android + run: | + # Install from a local file: URL installs land in a cache dir on + # some SwiftPM versions and the SDK then isn't found by + # `swift build --swift-sdk `; local-file installs + # register in ~/.swiftpm/swift-sdks. + curl -sfL --retry 3 -o /tmp/android-sdk.tar.gz \ + "https://download.swift.org/swift-${SWIFT_VERSION}-release/android-sdk/swift-${SWIFT_VERSION}-RELEASE/swift-${SWIFT_VERSION}-RELEASE_android.artifactbundle.tar.gz" + echo "${ANDROID_SDK_CHECKSUM} /tmp/android-sdk.tar.gz" | sha256sum -c - + swift sdk install /tmp/android-sdk.tar.gz --checksum "$ANDROID_SDK_CHECKSUM" + bundle="$HOME/.swiftpm/swift-sdks/swift-${SWIFT_VERSION}-RELEASE_android.artifactbundle" + test -d "$bundle" || bundle=$(find "$HOME" -maxdepth 6 -type d -name "swift-${SWIFT_VERSION}-RELEASE_android.artifactbundle" 2>/dev/null | head -1) + test -n "$bundle" || { echo "SDK bundle not found" >&2; exit 1; } + echo "ANDROID_SWIFT_SDK=$bundle" >> "$GITHUB_ENV" + # Populate the SDK's ndk-sysroot from the NDK (the bundle's + # setup-android-sdk.sh hardlinks it in). + (cd "$bundle/swift-android" && bash scripts/setup-android-sdk.sh) + - name: Cross-build native libraries for Android + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends autoconf automake libtool + Android/build-native-libs.sh "$ANDROID_SWIFT_SDK/swift-android" + - name: Use bionic-guarded forks of xtool-core and unxip + # TEMPORARY until xtool-org/xtool-core#2 is released and + # saagarjha/unxip#41 is merged: Superutils needs the Android + # guards for bionic Foundation, and unxip's zlib/getopt shims + # clash with the NDK's own modules when cross compiling. + # Mirrors alone don't retarget version-pinned deps, so also + # rewrite the unxip pin to the fork's 3.3 (which has the + # .when(platforms:) manifest). + run: | + mkdir -p ~/.swiftpm/configuration + cat > ~/.swiftpm/configuration/mirrors.json <<'EOF' + { + "object": [ + { + "original": "https://github.com/xtool-org/xtool-core", + "mirror": "https://github.com/hpr/xtool-core" + }, + { + "original": "https://github.com/saagarjha/unxip", + "mirror": "https://github.com/hpr/unxip" + } + ], + "version": 1 + } + EOF + python3 - <<'EOF2' + import json + p = json.load(open('Package.resolved')) + for pin in p['pins']: + if pin['identity'] == 'unxip': + pin['state']['revision'] = '7de3610da39c7cfa2635affab52fba169a94f4b4' + json.dump(p, open('Package.resolved', 'w'), indent=2) + EOF2 + # `swift package update unxip` recomputes the version and would + # overwrite the rewritten pin; only update xtool-core here, then + # re-apply the unxip pin afterwards (see below). + swift package update xtool-core + - name: Cross-compile for Android + run: | + # Re-apply the fork pins AFTER any resolution (above): + # `swift package update` recomputes versions from the canonical + # (cached) repos and overwrites rewritten pins. Pinned-revision + # checkouts go through the mirror, which serves the fork tags. + python3 - <<'EOF3' + import json + p = json.load(open('Package.resolved')) + for pin in p['pins']: + if pin['identity'] == 'unxip': + pin['state']['revision'] = '7de3610da39c7cfa2635affab52fba169a94f4b4' + if pin['identity'] == 'xtool-core': + pin['state']['revision'] = '58d5b679fa93f6e2a81901a54f43686ef3f53264' + json.dump(p, open('Package.resolved', 'w'), indent=2) + EOF3 + # SwiftPM's systemLibrary targets (xtool-core: openssl, + # libplist-2.0, ...) query pkg-config for cflags/libs. Without + # this, the host pkg-config resolves host (x86_64) .pc files and + # -L/usr/lib/x86_64-linux-gnu leaks into the link ahead of the + # NDK paths; lld then reads Ubuntu's libm.so *linker script*, + # whose OUTPUT_FORMAT(elf64-x86-64) silently overrides the -m + # emulation (lld/ELF/ScriptParser.cpp readOutputFormat), dropping + # the link to x86-64 so the Android-only --fix-cortex-a53-843419 + # is rejected. Point pkg-config at the .pc files generated by + # build-native-libs.sh so only the cross sysroot is visible. + export PKG_CONFIG_PATH="$ANDROID_SWIFT_SDK/swift-android/pkgconfig" + export PKG_CONFIG_LIBDIR="$ANDROID_SWIFT_SDK/swift-android/pkgconfig" + # The Android SDK registers API-level-suffixed triples, not the + # bare aarch64-unknown-linux-android. + swift build --product xtool --swift-sdk aarch64-unknown-linux-android28 + # Sanity-check the product: must be an aarch64 Android ELF. + file .build/aarch64-unknown-linux-android28/debug/xtool diff --git a/Android/build-native-libs.sh b/Android/build-native-libs.sh new file mode 100755 index 00000000..09f29365 --- /dev/null +++ b/Android/build-native-libs.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Cross-builds the native libraries xtool links against (OpenSSL and the +# libimobiledevice stack) for aarch64 Android, installing them into the +# Swift SDK for Android's NDK sysroot so that +# swift build --swift-sdk aarch64-unknown-linux-android28 +# can compile and link against them. +# +# Usage: Android/build-native-libs.sh +# ANDROID_NDK_HOME must point at an unpacked NDK (>= r27). +# +# The library set mirrors the Linux Docker image (see Dockerfile): OpenSSL +# plus libplist/libimobiledevice-glue/libusbmuxd/libtatsu/libimobiledevice +# from the libimobiledevice project, all built statically. libxadi is not +# needed: XADIProvider is os(Linux)-only (on macOS/Android anisette uses +# Omnisette), so the XADI system library never enters the link. +set -euo pipefail + +API=28 +TRIPLE=aarch64-linux-android +SDK=${1:?usage: build-native-libs.sh } +: "${ANDROID_NDK_HOME:?ANDROID_NDK_HOME must be set}" + +TOOLCHAIN=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin +export PATH="$TOOLCHAIN:$PATH" +export CC="$TRIPLE$API-clang" +export CXX="$TRIPLE$API-clang++" +export AR=llvm-ar RANLIB=llvm-ranlib STRIP=llvm-strip +export ANDROID_NDK_ROOT=$ANDROID_NDK_HOME + +WORK=$(mktemp -d) +PREFIX=$WORK/prefix +mkdir -p "$PREFIX" +# Point pkg-config exclusively at the cross prefix so the autotools builds +# find each other instead of the host's libraries. +export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig +export PKG_CONFIG_LIBDIR=$PREFIX/lib/pkgconfig +# Make all configure probes (not just pkg-config ones) find the prefix: +# AC_CHECK_LIB link tests need -L, header checks need -I. +export CPPFLAGS="-I$PREFIX/include" +export LDFLAGS="-L$PREFIX/lib" + +fetch() { + curl -sfL --retry 3 -o "$WORK/$2" "$1" +} + +echo "==> OpenSSL" +fetch \ + https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz \ + openssl.tar.gz +tar -C "$WORK" -xzf "$WORK/openssl.tar.gz" +( + cd "$WORK/openssl-3.3.2" + ./Configure android-arm64 -D__ANDROID_API__=$API no-shared no-tests \ + --prefix="$PREFIX" + make -j"$(nproc)" build_libs + make install_dev +) + +build_autotools() { # [configure args...] + local url=$1 dir=$2 + shift 2 + fetch "$url" "$dir.tar" + tar -C "$WORK" -xf "$WORK/$dir.tar" + ( + cd "$WORK/$dir" + ./configure --host="$TRIPLE" --prefix="$PREFIX" "$@" + make -j"$(nproc)" install + ) +} + +# bionic's pthreads are in libc and modern NDKs ship no libpthread; +# provide an empty static lib so -lpthread probes and links resolve. +"$AR" cr "$PREFIX/lib/libpthread.a" + +echo "==> libimobiledevice stack" +build_autotools \ + https://github.com/libimobiledevice/libplist/releases/download/2.6.0/libplist-2.6.0.tar.bz2 \ + libplist-2.6.0 --without-cython +build_autotools \ + https://github.com/libimobiledevice/libimobiledevice-glue/releases/download/1.3.1/libimobiledevice-glue-1.3.1.tar.bz2 \ + libimobiledevice-glue-1.3.1 +build_autotools \ + https://github.com/libimobiledevice/libusbmuxd/releases/download/2.1.0/libusbmuxd-2.1.0.tar.bz2 \ + libusbmuxd-2.1.0 --without-udev +# libtatsu and libimobiledevice need libcurl (they talk to Apple's TSS +# and activation servers), so build it before them; curl needs zlib, +# which the NDK does not ship pkg-config files for. +echo "==> zlib" +fetch https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz zlib.tar.gz +tar -C "$WORK" -xzf "$WORK/zlib.tar.gz" +( + cd "$WORK/zlib-1.3.1" + # position-independent, like everything else we build + CHOST="$TRIPLE" CFLAGS="-fPIC" ./configure --prefix="$PREFIX" --static + make -j"$(nproc)" install +) +build_autotools \ + https://github.com/curl/curl/releases/download/curl-8_16_0/curl-8.16.0.tar.bz2 \ + curl-8.16.0 --disable-shared --enable-static --with-openssl --without-libpsl \ + --without-libidn2 --without-brotli --without-zstd --without-nghttp2 \ + --disable-ldap --disable-ldaps --with-ca-bundle=/system/etc/security/cacerts +build_autotools \ + https://github.com/libimobiledevice/libtatsu/releases/download/1.0.4/libtatsu-1.0.4.tar.bz2 \ + libtatsu-1.0.4 +# libimobiledevice has no release tarball with the API SwiftyMobileDevice +# needs; use master like the Linux Docker image does. +fetch \ + https://codeload.github.com/libimobiledevice/libimobiledevice/tar.gz/refs/heads/master \ + libimobiledevice.tar.gz +tar -C "$WORK" -xzf "$WORK/libimobiledevice.tar.gz" +( + cd "$WORK/libimobiledevice-master" + # git-archive tarballs have no version info; provide one for bootstrap + git init -q . && git add -A && git -c user.email=ci@localhost -c user.name=ci commit -qm "libimobiledevice master snapshot" + echo "2.0.1-git" > .tarball-version + ./autogen.sh --host="$TRIPLE" --prefix="$PREFIX" --without-cython + make -j"$(nproc)" install +) + +# unxip links liblzma; build it too. +echo "==> xz" +fetch https://github.com/tukaani-project/xz/releases/download/v5.6.4/xz-5.6.4.tar.gz xz.tar +tar -C "$WORK" -xf "$WORK/xz.tar" +( + cd "$WORK/xz-5.6.4" + ./configure --host="$TRIPLE" --prefix="$PREFIX" --disable-shared --enable-static + make -j"$(nproc)" install +) + +echo "==> installing into SDK sysroot" +INC_DST=$SDK/ndk-sysroot/usr/include +LIB_DST=$SDK/ndk-sysroot/usr/lib/$TRIPLE +mkdir -p "$INC_DST" "$LIB_DST" +cp -R "$PREFIX/include/." "$INC_DST/" +cp -a "$PREFIX/lib/"*.a "$LIB_DST/" + +# Generate pkg-config files pointing at the sysroot. SwiftPM's +# systemLibrary targets query pkg-config for cflags/libs; on this host +# pkg-config would otherwise resolve to host (x86_64) libraries. +echo "==> generating pkg-config files" +PC_DST=$SDK/pkgconfig +mkdir -p "$PC_DST" +pc() { # [requires] + cat > "$PC_DST/$1.pc" < "$LIB_DST/libstdc++.so" + +# The NDK's prebuilt static libraries (libc.a & co.) carry zstd-compressed +# debug sections, but the swift.org toolchain's lld is built without zstd +# support and errors out reading them ("is compressed with ELFCOMPRESS_ZSTD, +# but lld is not built with zstd support"). Strip debug sections from every +# static archive in the sysroot so lld can consume them. Note -L: the SDK's +# setup-android-sdk.sh symlinks ndk-sysroot/usr/lib/ into the NDK by +# default (SWIFT_ANDROID_NDK_LINK=1), and find's default -P won't traverse it. +# Skip failures: some NDK "archives" (e.g. libc++.a in the per-API dirs) are +# GNU ld scripts, not objects, and llvm-strip can't parse them. +find -L "$SDK/ndk-sysroot" -name '*.a' -print0 | + while IFS= read -r -d '' a; do + llvm-strip --strip-debug "$a" 2>/dev/null || true + done +# Verify the strip actually took: fail here, not at the final Swift link. +archives=$(find -L "$SDK/ndk-sysroot" -name '*.a' | wc -l) +libc="$SDK/ndk-sysroot/usr/lib/$TRIPLE/libc.a" +remaining=$(readelf -SW "$libc" | grep -E '^[[:space:]]+\[[[:space:]0-9]+\]' | grep -c ' C ' || true) +echo "stripped debug sections from $archives archives; compressed sections left in $libc: $remaining" +[ "$remaining" = 0 ] || { echo "ERROR: zstd-compressed sections remain in $libc" >&2; exit 1; } + +echo "==> done: native libs installed into $SDK" diff --git a/Package.resolved b/Package.resolved index 0bcfc149..bab42f9e 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "80db0cddb0e78cf6a4c79f94537cdfebd8004679161532fbae8a57ee9d1d8cd1", + "originHash" : "41970e2ef95ed2d95550d89758de9edde764c927d6316551e31f0e6b2c05a55a", "pins" : [ { "identity" : "aexml", @@ -55,6 +55,15 @@ "version" : "6.2.0" } }, + { + "identity" : "opencombine", + "kind" : "remoteSourceControl", + "location" : "https://github.com/OpenCombine/OpenCombine.git", + "state" : { + "revision" : "8576f0d579b27020beccbccc3ea6844f3ddfc2c2", + "version" : "0.14.0" + } + }, { "identity" : "pathkit", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 8ea98c7f..dfdd6782 100644 --- a/Package.swift +++ b/Package.swift @@ -100,7 +100,7 @@ let package = Package( .product( name: "Subprocess", package: "swift-subprocess", - condition: .when(platforms: [.linux, .macOS]) + condition: .when(platforms: [.linux, .macOS, .android]) ), ] ), @@ -110,7 +110,7 @@ let package = Package( "DeveloperAPI", "CXKit", "XUtils", - .byName(name: "XADI", condition: .when(platforms: [.linux])), + .byName(name: "XADI", condition: .when(platforms: [.linux, .android])), .product(name: "ConcurrencyExtras", package: "swift-concurrency-extras"), .product(name: "Dependencies", package: "swift-dependencies"), .product(name: "SwiftyMobileDevice", package: "SwiftyMobileDevice"), @@ -128,17 +128,17 @@ let package = Package( .product( name: "OpenAPIAsyncHTTPClient", package: "swift-openapi-async-http-client", - condition: .when(platforms: [.linux]) + condition: .when(platforms: [.linux, .android]) ), .product( name: "AsyncHTTPClient", package: "async-http-client", - condition: .when(platforms: [.linux]) + condition: .when(platforms: [.linux, .android]) ), .product( name: "WebSocketKit", package: "websocket-kit", - condition: .when(platforms: [.linux]) + condition: .when(platforms: [.linux, .android]) ), ], cSettings: cSettings diff --git a/Sources/XKit/HTTPClientProtocol/AsyncHTTPClient+HTTP.swift b/Sources/XKit/HTTPClientProtocol/AsyncHTTPClient+HTTP.swift index eaa8f607..bb717312 100644 --- a/Sources/XKit/HTTPClientProtocol/AsyncHTTPClient+HTTP.swift +++ b/Sources/XKit/HTTPClientProtocol/AsyncHTTPClient+HTTP.swift @@ -5,7 +5,7 @@ // Created by Kabir Oberai on 05/05/21. // -#if os(Linux) +#if os(Linux) || os(Android) import Foundation import AsyncHTTPClient import NIO diff --git a/Sources/XKit/HTTPClientProtocol/URLSession+HTTP.swift b/Sources/XKit/HTTPClientProtocol/URLSession+HTTP.swift index 74527bfc..b0c3eda7 100644 --- a/Sources/XKit/HTTPClientProtocol/URLSession+HTTP.swift +++ b/Sources/XKit/HTTPClientProtocol/URLSession+HTTP.swift @@ -5,7 +5,7 @@ // Created by Kabir Oberai on 05/05/21. // -#if !os(Linux) +#if !os(Linux) && !os(Android) import Foundation import ConcurrencyExtras import OpenAPIRuntime diff --git a/Sources/XKit/Utilities/CHelpers.swift b/Sources/XKit/Utilities/CHelpers.swift index afd1e8ea..3b89cbde 100644 --- a/Sources/XKit/Utilities/CHelpers.swift +++ b/Sources/XKit/Utilities/CHelpers.swift @@ -8,10 +8,20 @@ import Foundation import CXKit +#if os(Android) +import Android +#endif +#if os(Android) +// bionic's FILE is an incomplete type, imported as OpaquePointer +package var stdoutSafe: OpaquePointer { + get_stdout() +} +#else package var stdoutSafe: UnsafeMutablePointer { get_stdout() } +#endif extension Data { init?(deallocator: Deallocator = .free, acceptor: (inout Int) -> UnsafeMutableRawPointer?) { diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 96e7f6f6..2df92a4b 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -305,6 +305,9 @@ struct SDKBuilder { } print() + print("[Validating SDKs]") + try Self.validateInstalledSDKs(in: dev) + print("[Cleaning up]") if let cleanupStageDir { try? FileManager.default.removeItem(at: cleanupStageDir) @@ -357,6 +360,37 @@ struct SDKBuilder { return dev } + /// Files expected in every installed SDK. When the XIP extractor + /// fails to recreate hard-linked duplicates (e.g. unxip's linkat(2) + /// failing on hosts that restrict hard links, such as SELinux- + /// enforced Android), these files silently vanish from the staged + /// tree; the resulting SDK then produces confusing compile errors + /// ("'__config' file not found") long after installation. Validating + /// a sample of known victims turns that silent corruption into a + /// clear failure at install time. + private static let sdkValidationPaths = [ + // libc++ headers are stored as hardlink duplicates in the XIP + "Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/include/c++/v1/__config", + "Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/include/c++/v1/vector", + "Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/include/c++/v1/string", + // so are the Swift runtime shims headers + "Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib/swift/shims/HeapObject.h", + "Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib/swift/shims/KeyPath.h", + ] + + private static func validateInstalledSDKs(in dev: URL) throws { + let manager = FileManager.default + for path in sdkValidationPaths + where !manager.fileExists(atPath: dev.appending(path: path).path) { + throw Console.Error(""" + SDK validation failed: \(path) is missing. The SDK bundle is \ + corrupt; this can happen when XIP extraction silently fails \ + to recreate hard-linked files. Please try installing the SDK \ + again (and report a bug if it persists). + """) + } + } + // returns the number of files we actually want to keep, // useful for computing progress % during fs traversal private func extractXIP(inputPath: String, outDir: String) async throws -> Int { diff --git a/Sources/XToolSupport/XTool.swift b/Sources/XToolSupport/XTool.swift index 2cec336e..1f86969d 100644 --- a/Sources/XToolSupport/XTool.swift +++ b/Sources/XToolSupport/XTool.swift @@ -1,4 +1,7 @@ import Foundation +#if os(Android) +import Android +#endif import XKit import ArgumentParser import XUtils @@ -74,7 +77,7 @@ extension ParsableCommand where Self: SendableMetatype { } } - signal(SIGINT, SIG_IGN) + _ = signal(SIGINT, SIG_IGN) let source = DispatchSource.makeSignalSource(signal: SIGINT) source.setEventHandler { task.cancel() } source.resume() diff --git a/Sources/XUtils/System+Utils.swift b/Sources/XUtils/System+Utils.swift index df47da9f..530f030c 100644 --- a/Sources/XUtils/System+Utils.swift +++ b/Sources/XUtils/System+Utils.swift @@ -1,4 +1,7 @@ import Foundation +#if os(Android) +import Android +#endif #if canImport(System) import System