From 0214f6fb5e804d7c6099c05fb9fe2260dc7317ae Mon Sep 17 00:00:00 2001 From: tobtoht Date: Wed, 23 Feb 2022 17:04:23 +0100 Subject: [PATCH] Rework Tor bundling --- .gitignore | 7 ++-- BUILDING.md | 2 +- CMakeLists.txt | 29 +++++++------ Dockerfile.linux | 10 +++-- Dockerfile.windows | 70 ++++++++----------------------- Makefile | 4 +- cmake/TorQrcGenerator.cmake | 19 +++++++++ {src => cmake}/assets_tor.qrc | 2 +- src/CMakeLists.txt | 12 +----- src/assets/{exec => tor}/.gitkeep | 0 src/assets_tor_macos.qrc | 6 --- src/main.cpp | 9 ++-- src/utils/TorManager.cpp | 55 ++++++++++-------------- 13 files changed, 95 insertions(+), 130 deletions(-) create mode 100644 cmake/TorQrcGenerator.cmake rename {src => cmake}/assets_tor.qrc (69%) rename src/assets/{exec => tor}/.gitkeep (100%) delete mode 100644 src/assets_tor_macos.qrc diff --git a/.gitignore b/.gitignore index 445ab4e..0e8b557 100644 --- a/.gitignore +++ b/.gitignore @@ -5,13 +5,14 @@ cmake-build-debug/* *.user *.stash build/* +build*/ CMakeCache.txt CMakeFiles cmake_install.cmake feather_autogen/ feather.cbp -src/tor/* -!src/tor/.gitkeep src/config-feather.h -src/assets/exec/* +src/assets_tor.qrc feather.AppDir/* +src/assets/tor/* +!src/assets/tor/.gitkeep diff --git a/BUILDING.md b/BUILDING.md index dbe19cd..231a8cb 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -137,7 +137,7 @@ Building the base image takes a while. You only need to build the base image onc #### 3. Build ```bash -docker run --rm -it -v $PWD:/feather -w /feather feather:win sh -c 'make depends root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j4' +docker run --rm -it -v $PWD:/feather -w /feather feather:win sh -c 'make release-static-windows root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j4' ``` If you're re-running a build make sure to `rm -rf build/` first. diff --git a/CMakeLists.txt b/CMakeLists.txt index 0553878..9bd8c2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.20) project(feather) message(STATUS "Initiating compile using CMake ${CMAKE_VERSION}") @@ -31,7 +31,7 @@ if(DEBUG) set(CMAKE_VERBOSE_MAKEFILE ON) endif() -set(MONERO_HEAD "6a2b96394d3c81a4ccf9be0daea02afe5f6f3683") +set(MONERO_HEAD "4c2fa8df7f82ab00ec5d58329f4b8314ae1038d3") set(BUILD_GUI_DEPS ON) option(ARCH "Target architecture" "x86-64") set(BUILD_64 ON) @@ -153,31 +153,30 @@ if("$ENV{DRONE}" STREQUAL "true") message(STATUS "We are inside a static compile with Drone CI") endif() -# To build Feather with embedded (and static) Tor, pass CMake -DTOR_BIN=/path/to/tor -if(TOR_BIN) - if(APPLE) - execute_process(COMMAND bash -c "touch ${CMAKE_CURRENT_SOURCE_DIR}/src/tor/libevent-2.1.7.dylib") +include(TorQrcGenerator) + +# To build Feather with embedded (and static) Tor, pass CMake -DTOR_DIR=/path/to/tor/ +if(TOR_DIR) + if (NOT TOR_VERSION) + message(FATAL_ERROR "TOR_DIR is specified but TOR_VERSION is not set") endif() message(STATUS "Embedded Tor version: ${TOR_VERSION}") configure_file("cmake/config-feather.h.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/src/config-feather.h") - # on the buildbot Tor is baked into the image - # - linux: See `Dockerfile` - # - windows: See `Dockerfile.windows` - # - macos: taken from Tor Browser official release - if(REPRODUCIBLE) # Always copy Tor when doing a reproducible build to prevent old versions from getting included - set(TOR_COPY_CMD "cp ${TOR_BIN} ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/exec/tor") + # Always copy Tor when doing a reproducible build to prevent old versions from getting included + if (REPRODUCIBLE) + set(TOR_COPY_CMD "cp -a ${TOR_DIR}/* ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/tor") else() - set(TOR_COPY_CMD "cp -u ${TOR_BIN} ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/exec/tor") + set(TOR_COPY_CMD "cp -au ${TOR_DIR}/* ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/tor") endif() - message(STATUS "${TOR_COPY_CMD}") + execute_process(COMMAND bash -c "${TOR_COPY_CMD}" RESULT_VARIABLE ret) if(ret EQUAL "1") message(FATAL_ERROR "Tor copy failure: ${TOR_COPY_CMD}") endif() - message(STATUS "Embedding Tor binary at ${TOR_BIN}") + message(STATUS "Embedding Tor binaries at ${TOR_DIR}") else() message(STATUS "Skipping Tor inclusion because -DTOR_BIN=Off") endif() diff --git a/Dockerfile.linux b/Dockerfile.linux index f5efbe2..d5cc999 100644 --- a/Dockerfile.linux +++ b/Dockerfile.linux @@ -36,9 +36,9 @@ RUN git clone -b OpenSSL_1_1_1m --depth 1 https://github.com/openssl/openssl.git rm -rf $(pwd) # CMake: Required to build libqrencode, monero-seed, libzip -RUN git clone -b v3.18.4 --depth 1 https://github.com/Kitware/CMake && \ +RUN git clone -b v3.22.2 --depth 1 https://github.com/Kitware/CMake && \ cd CMake && \ - git reset --hard 3cc3d42aba879fff5e85b363ae8f21386a3f9f9b && \ + git reset --hard 8428e39ed9cddb3b7f1a6f7a58cb8617503183d2 && \ ./bootstrap && \ make -j$THREADS && \ make -j$THREADS install && \ @@ -227,9 +227,11 @@ RUN git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib && \ make -j$THREADS install && \ rm -rf $(pwd) -# Tor: Optional for Feather (-DTOR_BIN) +# Tor: Optional for Feather (-DTOR_DIR) # Binary can be embedded in Feather -ENV TOR_BIN=/usr/local/tor/bin/tor +# TOR_VERSION is used by CMAKE to determine embedded version +ENV TOR_VERSION=0.4.6.9 +ENV TOR_DIR=/usr/local/tor/bin/tor RUN git clone -b tor-0.4.6.9 --depth 1 https://git.torproject.org/tor.git && \ cd tor && \ git reset --hard ea2ada6d1459f829446b6b1e66c557d1b084e78b && \ diff --git a/Dockerfile.windows b/Dockerfile.windows index 4545607..fc694e0 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -10,11 +10,10 @@ ENV CHECK_UPDATES=ON ENV WITH_SCANNER=ON ENV OPENSSL_ROOT_DIR=/usr/local/openssl/ -ENV TOR_BIN=/usr/local/tor/bin/tor.exe RUN apt update && \ apt install -y \ - curl wget zip automake build-essential cmake gcc-mingw-w64 g++-mingw-w64 gettext git libtool pkg-config \ + curl wget zip automake build-essential gcc-mingw-w64 g++-mingw-w64 gettext git libtool pkg-config \ python \ # zbar autopoint && \ @@ -23,6 +22,15 @@ RUN apt update && \ RUN update-alternatives --set x86_64-w64-mingw32-g++ $(which x86_64-w64-mingw32-g++-posix) && \ update-alternatives --set x86_64-w64-mingw32-gcc $(which x86_64-w64-mingw32-gcc-posix) +# The version from ubuntu repo is too old +RUN git clone -b v3.22.2 --depth 1 https://github.com/Kitware/CMake && \ + cd CMake && \ + git reset --hard 8428e39ed9cddb3b7f1a6f7a58cb8617503183d2 && \ + ./bootstrap -- -DCMAKE_USE_OPENSSL=OFF && \ + make -j$THREADS && \ + make -j$THREADS install && \ + rm -rf $(pwd) + RUN git clone -b v0.17.2.0 --depth 1 https://github.com/monero-project/monero && \ cd monero && \ git reset --hard f6e63ef260e795aacd408c28008398785b84103a && \ @@ -90,7 +98,7 @@ RUN git clone -b libgcrypt-1.8.5 --depth 1 git://git.gnupg.org/libgcrypt.git && cd .. && \ rm -rf libgcrypt -# zlib -> libpng, Tor +# zlib -> libpng RUN git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib && \ cd zlib && \ git reset --hard cacf7f1d4e3d44d871b605da3b647f07d718623f && \ @@ -129,55 +137,13 @@ RUN wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.16.tar.gz && \ make -j$THREADS install && \ rm -rf $(pwd) -# OpenSSL -> Tor -RUN wget https://www.openssl.org/source/openssl-1.1.1m.tar.gz && \ - echo "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 openssl-1.1.1m.tar.gz" | sha256sum -c && \ - tar -xzf openssl-1.1.1l.tar.gz && \ - rm openssl-1.1.1l.tar.gz && \ - cd openssl-1.1.1l && \ - ./Configure mingw64 no-shared no-dso --cross-compile-prefix=x86_64-w64-mingw32- --prefix=/usr/local/openssl && \ - make -j$THREADS && \ - make -j$THREADS install_sw && \ - rm -rf $(pwd) - -# libevent -> Tor -RUN wget https://github.com/libevent/libevent/releases/download/release-2.1.11-stable/libevent-2.1.11-stable.tar.gz && \ - echo "a65bac6202ea8c5609fd5c7e480e6d25de467ea1917c08290c521752f147283d libevent-2.1.11-stable.tar.gz" | sha256sum -c && \ - tar -zxvf libevent-2.1.11-stable.tar.gz && \ - cd libevent-2.1.11-stable && \ - ./configure --prefix=/usr/local/libevent \ - --disable-shared \ - --enable-static \ - --with-pic \ - --host=x86_64-w64-mingw32 && \ - make -j$THREADS && \ - make -j$THREADS install && \ - rm -rf $(pwd) - -RUN git clone -b tor-0.4.6.9 --depth 1 https://git.torproject.org/tor.git && \ - cd tor && \ - git reset --hard ea2ada6d1459f829446b6b1e66c557d1b084e78b && \ - ./autogen.sh && \ - ./configure --host=x86_64-w64-mingw32 \ - --disable-asciidoc \ - --disable-zstd \ - --disable-lzma \ - --disable-manpage \ - --disable-html-manual \ - --disable-system-torrc \ - --disable-module-relay \ - --enable-static-tor \ - --with-libevent-dir=/usr/local/libevent \ - --with-openssl-dir=/usr/local/openssl \ - --with-zlib-dir=/usr/x86_64-w64-mingw32 \ - --disable-tool-name-check \ - --enable-fatal-warnings \ - --prefix=/usr/local/tor \ - LIBS=-lcrypt32 && \ - make -j$THREADS && \ - make -j$THREADS install && \ - rm -rf $(pwd) && \ - strip -s -D /usr/local/tor/bin/tor.exe +# TOR_VERSION is used by CMAKE to determine embedded version +ENV TOR_VERSION=0.4.6.9 +ENV TOR_DIR=/tor/Tor/ +RUN wget https://dist.torproject.org/torbrowser/11.0.6/tor-win64-0.4.6.9.zip && \ + echo "fabb46fb0b82c7cd3592c96554ffd8c0766b2e467559e3613b3e14ac191aed19 tor-win64-0.4.6.9.zip" | sha256sum -c && \ + unzip tor-win64-0.4.6.9.zip -d tor && \ + rm tor-win64-0.4.6.9.zip RUN git clone https://github.com/nih-at/libzip.git && \ cd libzip && \ diff --git a/Makefile b/Makefile index f461f82..7545481 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: 2020-2022 The Monero Project CMAKEFLAGS = \ - -DTOR_BIN=$(or ${TOR_BIN}, Off) \ + -DTOR_DIR=$(or ${TOR_DIR}, Off) \ -DTOR_VERSION=$(or ${TOR_VERSION}, Off) \ -DCHECK_UPDATES=$(or ${CHECK_UPDATES}, Off) \ -DWITH_SCANNER=$(or ${WITH_SCANNER}, Off) \ @@ -56,7 +56,7 @@ release-static-linux-arm64-rpi: ../.. && \ $(MAKE) -depends: +release-static-windows: mkdir -p build/$(target)/release && \ cd build/$(target)/release && \ cmake \ diff --git a/cmake/TorQrcGenerator.cmake b/cmake/TorQrcGenerator.cmake new file mode 100644 index 0000000..54687b9 --- /dev/null +++ b/cmake/TorQrcGenerator.cmake @@ -0,0 +1,19 @@ +if (TOR_DIR) + FILE(GLOB TOR_FILES LIST_DIRECTORIES false ${TOR_DIR}/*) + + foreach(FILE ${TOR_FILES}) + cmake_path(GET FILE FILENAME FILE_REL) + list(APPEND QRC_LIST " assets/tor/${FILE_REL}") + + if (FILE_REL STREQUAL "tor" OR FILE_REL STREQUAL "tor.exe") + set(TOR_BIN_FOUND 1) + endif() + endforeach() + + if (NOT TOR_BIN_FOUND) + message(FATAL_ERROR "TOR_DIR was specified but the Tor binary could not be found") + endif() +endif() + +list(JOIN QRC_LIST "\n" QRC_DATA) +configure_file("cmake/assets_tor.qrc" "${CMAKE_CURRENT_SOURCE_DIR}/src/assets_tor.qrc") \ No newline at end of file diff --git a/src/assets_tor.qrc b/cmake/assets_tor.qrc similarity index 69% rename from src/assets_tor.qrc rename to cmake/assets_tor.qrc index 7f0c2cf..f5363d5 100644 --- a/src/assets_tor.qrc +++ b/cmake/assets_tor.qrc @@ -1,5 +1,5 @@ - assets/exec/tor +@QRC_DATA@ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 624987b..3c6461d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,7 +25,7 @@ find_package(Qt5 REQUIRED COMPONENTS ${QT5_COMPONENTS}) add_subdirectory(openpgp) -qt5_add_resources(RESOURCES assets.qrc) +qt5_add_resources(RESOURCES assets.qrc assets_tor.qrc) # Compile source files (.h/.cpp) file(GLOB SOURCE_FILES @@ -70,14 +70,6 @@ if (WITH_SCANNER) ${SCANNER_FILES}) endif() -if(TOR_BIN) - if(APPLE) - set(ASSETS_TOR "assets_tor_macos.qrc") - else() - set(ASSETS_TOR "assets_tor.qrc") - endif() -endif() - set(EXECUTABLE_FLAG) if(MINGW) set(EXECUTABLE_FLAG WIN32) @@ -164,7 +156,7 @@ if(LOCALMONERO) target_compile_definitions(feather PRIVATE HAS_LOCALMONERO=1) endif() -if(TOR_BIN) +if(TOR_DIR) target_compile_definitions(feather PRIVATE HAS_TOR_BIN=1) endif() diff --git a/src/assets/exec/.gitkeep b/src/assets/tor/.gitkeep similarity index 100% rename from src/assets/exec/.gitkeep rename to src/assets/tor/.gitkeep diff --git a/src/assets_tor_macos.qrc b/src/assets_tor_macos.qrc deleted file mode 100644 index 909796c..0000000 --- a/src/assets_tor_macos.qrc +++ /dev/null @@ -1,6 +0,0 @@ - - - assets/exec/tor - assets/exec/libevent-2.1.7.dylib - - diff --git a/src/main.cpp b/src/main.cpp index 35afc2d..531fe90 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,9 +25,7 @@ int main(int argc, char *argv[]) { Q_INIT_RESOURCE(assets); -#if defined(Q_OS_MAC) && defined(HAS_TOR_BIN) - Q_INIT_RESOURCE(assets_tor_macos); -#elif defined(HAS_TOR_BIN) +#if defined(HAS_TOR_BIN) Q_INIT_RESOURCE(assets_tor); #endif @@ -194,6 +192,11 @@ if (AttachConsole(ATTACH_PARENT_PROCESS)) { else info["Mode"] = "Mainnet"; info["SSL"] = QSslSocket::sslLibraryVersionString(); info["SSL build"] = QSslSocket::sslLibraryBuildVersionString(); +#if defined(TOR_VERSION) + info["Tor version"] = TOR_VERSION; +#else + info["Tor version"] = "Not bundled"; +#endif for (const QString &k: info.keys()) { qWarning().nospace().noquote() << QString("%1: %2").arg(k, info[k]); } diff --git a/src/utils/TorManager.cpp b/src/utils/TorManager.cpp index d34ea60..48c228a 100644 --- a/src/utils/TorManager.cpp +++ b/src/utils/TorManager.cpp @@ -157,35 +157,13 @@ void TorManager::handleProcessError(QProcess::ProcessError error) { } bool TorManager::unpackBins() { - QString torFile; - - // On MacOS write libevent to disk -#if defined(Q_OS_MAC) - QString libEvent = ":/assets/exec/libevent-2.1.7.dylib"; - if (Utils::fileExists(libEvent)) { - QFile e(libEvent); - QFileInfo eventInfo(e); - auto libEventPath = QDir(this->torDir).filePath(eventInfo.fileName()); - qDebug() << libEventPath; - e.copy(libEventPath); - e.close(); - } -#endif - - torFile = ":/assets/exec/tor"; - if (!Utils::fileExists(torFile)) - return false; - - // write to disk - QFile f(torFile); - QFileInfo fileInfo(f); - this->torPath = QDir(this->torDir).filePath(fileInfo.fileName()); - + QString torBin = "tor"; #if defined(Q_OS_WIN) - if(!this->torPath.endsWith(".exe")) - this->torPath += ".exe"; + torBin += ".exe"; #endif + this->torPath = QDir(this->torDir).filePath(torBin); + SemanticVersion embeddedVersion = SemanticVersion::fromString(QString(TOR_VERSION)); SemanticVersion filesystemVersion = this->getVersion(torPath); qDebug() << QString("Tor versions: embedded %1, filesystem %2").arg(embeddedVersion.toString(), filesystemVersion.toString()); @@ -194,18 +172,29 @@ bool TorManager::unpackBins() { QFile::setPermissions(torPath, QFile::ReadOther | QFile::WriteOther); if (!QFile::remove(torPath)) { qWarning() << "Unable to remove old Tor binary"; - }; + return false; + } } - qDebug() << "Writing Tor executable to " << this->torPath; - f.copy(torPath); - f.close(); + if (embeddedVersion > filesystemVersion) { + QDirIterator it(":/assets/tor", QDirIterator::Subdirectories); + while (it.hasNext()) { + QString assetFile = it.next(); + QFileInfo assetFileInfo = QFileInfo(assetFile); + QFile f(assetFile); + QString filePath = QDir(this->torDir).filePath(assetFileInfo.fileName()); + f.copy(filePath); + f.close(); + } + qInfo() << "Wrote Tor binaries to: " << this->torDir; + } #if defined(Q_OS_UNIX) - QFile torBin(this->torPath); - torBin.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther - | QFile::ReadOwner | QFile::ReadGroup | QFile::ReadOther); + QFile tor(this->torPath); + tor.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther + | QFile::ReadOwner | QFile::ReadGroup | QFile::ReadOther); #endif + return true; }