Rework Tor bundling

.gitignore

@@ -5,13 +5,14 @@ cmake-build-debug/*
 *.user
 *.stash
 build/*
+build*/
 CMakeCache.txt
 CMakeFiles
 cmake_install.cmake
 feather_autogen/
 feather.cbp
-src/tor/*
-!src/tor/.gitkeep
 src/config-feather.h
-src/assets/exec/*
+src/assets_tor.qrc
 feather.AppDir/*
+src/assets/tor/*
+!src/assets/tor/.gitkeep

BUILDING.md

@@ -137,7 +137,7 @@ Building the base image takes a while. You only need to build the base image onc
 #### 3. Build
 
 ```bash
-docker run --rm -it -v $PWD:/feather -w /feather feather:win sh -c 'make depends root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j4'
+docker run --rm -it -v $PWD:/feather -w /feather feather:win sh -c 'make release-static-windows root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j4'
 ```
 
 If you're re-running a build make sure to `rm -rf build/` first.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.13)
+cmake_minimum_required(VERSION 3.20)
 project(feather)
 
 message(STATUS "Initiating compile using CMake ${CMAKE_VERSION}")
@@ -31,7 +31,7 @@ if(DEBUG)
     set(CMAKE_VERBOSE_MAKEFILE ON)
 endif()
 
-set(MONERO_HEAD "6a2b96394d3c81a4ccf9be0daea02afe5f6f3683")
+set(MONERO_HEAD "4c2fa8df7f82ab00ec5d58329f4b8314ae1038d3")
 set(BUILD_GUI_DEPS ON)
 option(ARCH "Target architecture" "x86-64")
 set(BUILD_64 ON)
@@ -153,31 +153,30 @@ if("$ENV{DRONE}" STREQUAL "true")
     message(STATUS "We are inside a static compile with Drone CI")
 endif()
 
-# To build Feather with embedded (and static) Tor, pass CMake -DTOR_BIN=/path/to/tor
+include(TorQrcGenerator)
-if(TOR_BIN)
+
-    if(APPLE)
+# To build Feather with embedded (and static) Tor, pass CMake -DTOR_DIR=/path/to/tor/
-        execute_process(COMMAND bash -c "touch ${CMAKE_CURRENT_SOURCE_DIR}/src/tor/libevent-2.1.7.dylib")
+if(TOR_DIR)
+    if (NOT TOR_VERSION)
+        message(FATAL_ERROR "TOR_DIR is specified but TOR_VERSION is not set")
     endif()
 
     message(STATUS "Embedded Tor version: ${TOR_VERSION}")
     configure_file("cmake/config-feather.h.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/src/config-feather.h")
 
-    # on the buildbot Tor is baked into the image
+    # Always copy Tor when doing a reproducible build to prevent old versions from getting included
-    # - linux: See `Dockerfile`
+    if (REPRODUCIBLE)
-    # - windows: See `Dockerfile.windows`
+        set(TOR_COPY_CMD "cp -a ${TOR_DIR}/* ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/tor")
-    # - macos: taken from Tor Browser official release
-    if(REPRODUCIBLE) # Always copy Tor when doing a reproducible build to prevent old versions from getting included
-        set(TOR_COPY_CMD "cp ${TOR_BIN} ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/exec/tor")
     else()
-        set(TOR_COPY_CMD "cp -u ${TOR_BIN} ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/exec/tor")
+        set(TOR_COPY_CMD "cp -au ${TOR_DIR}/* ${CMAKE_CURRENT_SOURCE_DIR}/src/assets/tor")
     endif()
-    message(STATUS "${TOR_COPY_CMD}")
+
     execute_process(COMMAND bash -c "${TOR_COPY_CMD}" RESULT_VARIABLE ret)
     if(ret EQUAL "1")
         message(FATAL_ERROR "Tor copy failure: ${TOR_COPY_CMD}")
     endif()
 
-    message(STATUS "Embedding Tor binary at ${TOR_BIN}")
+    message(STATUS "Embedding Tor binaries at ${TOR_DIR}")
 else()
     message(STATUS "Skipping Tor inclusion because -DTOR_BIN=Off")
 endif()

Dockerfile.linux

@@ -36,9 +36,9 @@ RUN git clone -b OpenSSL_1_1_1m --depth 1 https://github.com/openssl/openssl.git
     rm -rf $(pwd)
 
 # CMake: Required to build libqrencode, monero-seed, libzip
-RUN git clone -b v3.18.4 --depth 1 https://github.com/Kitware/CMake && \
+RUN git clone -b v3.22.2 --depth 1 https://github.com/Kitware/CMake && \
     cd CMake && \
-    git reset --hard 3cc3d42aba879fff5e85b363ae8f21386a3f9f9b && \
+    git reset --hard 8428e39ed9cddb3b7f1a6f7a58cb8617503183d2 && \
     ./bootstrap && \
     make -j$THREADS && \
     make -j$THREADS install && \
@@ -227,9 +227,11 @@ RUN git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib && \
     make -j$THREADS install && \
     rm -rf $(pwd)
 
-# Tor: Optional for Feather (-DTOR_BIN)
+# Tor: Optional for Feather (-DTOR_DIR)
 # Binary can be embedded in Feather
-ENV TOR_BIN=/usr/local/tor/bin/tor
+# TOR_VERSION is used by CMAKE to determine embedded version
+ENV TOR_VERSION=0.4.6.9
+ENV TOR_DIR=/usr/local/tor/bin/tor
 RUN git clone -b tor-0.4.6.9 --depth 1 https://git.torproject.org/tor.git && \
     cd tor && \
     git reset --hard ea2ada6d1459f829446b6b1e66c557d1b084e78b && \

Dockerfile.windows

@@ -10,11 +10,10 @@ ENV CHECK_UPDATES=ON
 ENV WITH_SCANNER=ON
 
 ENV OPENSSL_ROOT_DIR=/usr/local/openssl/
-ENV TOR_BIN=/usr/local/tor/bin/tor.exe
 
 RUN apt update && \
     apt install -y \
-    curl wget zip automake build-essential cmake gcc-mingw-w64 g++-mingw-w64 gettext git libtool pkg-config \
+    curl wget zip automake build-essential gcc-mingw-w64 g++-mingw-w64 gettext git libtool pkg-config \
     python \
 # zbar
     autopoint && \
@@ -23,6 +22,15 @@ RUN apt update && \
 RUN update-alternatives --set x86_64-w64-mingw32-g++ $(which x86_64-w64-mingw32-g++-posix) && \
     update-alternatives --set x86_64-w64-mingw32-gcc $(which x86_64-w64-mingw32-gcc-posix)
 
+# The version from ubuntu repo is too old
+RUN git clone -b v3.22.2 --depth 1 https://github.com/Kitware/CMake && \
+    cd CMake && \
+    git reset --hard 8428e39ed9cddb3b7f1a6f7a58cb8617503183d2 && \
+    ./bootstrap -- -DCMAKE_USE_OPENSSL=OFF && \
+    make -j$THREADS && \
+    make -j$THREADS install && \
+    rm -rf $(pwd)
+
 RUN git clone -b v0.17.2.0 --depth 1 https://github.com/monero-project/monero && \
     cd monero && \
     git reset --hard f6e63ef260e795aacd408c28008398785b84103a && \
@@ -90,7 +98,7 @@ RUN git clone -b libgcrypt-1.8.5 --depth 1 git://git.gnupg.org/libgcrypt.git &&
     cd .. && \
     rm -rf libgcrypt
 
-# zlib -> libpng, Tor
+# zlib -> libpng
 RUN git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib && \
     cd zlib && \
     git reset --hard cacf7f1d4e3d44d871b605da3b647f07d718623f && \
@@ -129,55 +137,13 @@ RUN wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.16.tar.gz && \
     make -j$THREADS install && \
     rm -rf $(pwd)
 
-# OpenSSL -> Tor
+# TOR_VERSION is used by CMAKE to determine embedded version
-RUN wget https://www.openssl.org/source/openssl-1.1.1m.tar.gz && \
+ENV TOR_VERSION=0.4.6.9
-    echo "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 openssl-1.1.1m.tar.gz" | sha256sum -c && \
+ENV TOR_DIR=/tor/Tor/
-    tar -xzf openssl-1.1.1l.tar.gz && \
+RUN wget https://dist.torproject.org/torbrowser/11.0.6/tor-win64-0.4.6.9.zip && \
-    rm openssl-1.1.1l.tar.gz && \
+    echo "fabb46fb0b82c7cd3592c96554ffd8c0766b2e467559e3613b3e14ac191aed19 tor-win64-0.4.6.9.zip" | sha256sum -c && \
-    cd openssl-1.1.1l && \
+    unzip tor-win64-0.4.6.9.zip -d tor && \
-    ./Configure mingw64 no-shared no-dso --cross-compile-prefix=x86_64-w64-mingw32- --prefix=/usr/local/openssl && \
+    rm tor-win64-0.4.6.9.zip
-    make -j$THREADS && \
-    make -j$THREADS install_sw && \
-    rm -rf $(pwd)
-
-# libevent -> Tor
-RUN wget https://github.com/libevent/libevent/releases/download/release-2.1.11-stable/libevent-2.1.11-stable.tar.gz && \
-    echo "a65bac6202ea8c5609fd5c7e480e6d25de467ea1917c08290c521752f147283d libevent-2.1.11-stable.tar.gz" | sha256sum -c && \
-    tar -zxvf libevent-2.1.11-stable.tar.gz && \
-    cd libevent-2.1.11-stable && \
-    ./configure --prefix=/usr/local/libevent \
-                --disable-shared \
-                --enable-static \
-                --with-pic \
-                --host=x86_64-w64-mingw32 && \
-    make -j$THREADS && \
-    make -j$THREADS install && \
-    rm -rf $(pwd)
-
-RUN git clone -b tor-0.4.6.9 --depth 1 https://git.torproject.org/tor.git && \
-    cd tor  && \
-    git reset --hard ea2ada6d1459f829446b6b1e66c557d1b084e78b && \
-    ./autogen.sh && \
-    ./configure --host=x86_64-w64-mingw32 \
-    --disable-asciidoc \
-    --disable-zstd \
-    --disable-lzma \
-    --disable-manpage \
-    --disable-html-manual \
-    --disable-system-torrc \
-    --disable-module-relay \
-    --enable-static-tor \
-    --with-libevent-dir=/usr/local/libevent \
-    --with-openssl-dir=/usr/local/openssl \
-    --with-zlib-dir=/usr/x86_64-w64-mingw32 \
-    --disable-tool-name-check \
-    --enable-fatal-warnings \
-    --prefix=/usr/local/tor \
-    LIBS=-lcrypt32 && \
-    make -j$THREADS && \
-    make -j$THREADS install && \
-    rm -rf $(pwd) && \
-    strip -s -D /usr/local/tor/bin/tor.exe
 
 RUN git clone https://github.com/nih-at/libzip.git && \
     cd libzip && \

Makefile

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: 2020-2022 The Monero Project
 
 CMAKEFLAGS = \
-	-DTOR_BIN=$(or ${TOR_BIN}, Off) \
+	-DTOR_DIR=$(or ${TOR_DIR}, Off) \
 	-DTOR_VERSION=$(or ${TOR_VERSION}, Off) \
 	-DCHECK_UPDATES=$(or ${CHECK_UPDATES}, Off) \
 	-DWITH_SCANNER=$(or ${WITH_SCANNER}, Off) \
@@ -56,7 +56,7 @@ release-static-linux-arm64-rpi:
 		../.. && \
 	$(MAKE)
 
-depends:
+release-static-windows:
 	mkdir -p build/$(target)/release && \
 	cd build/$(target)/release && \
 	cmake \

cmake/TorQrcGenerator.cmake

@@ -0,0 +1,19 @@
+if (TOR_DIR)
+    FILE(GLOB TOR_FILES LIST_DIRECTORIES false ${TOR_DIR}/*)
+
+    foreach(FILE ${TOR_FILES})
+        cmake_path(GET FILE FILENAME FILE_REL)
+        list(APPEND QRC_LIST "        <file>assets/tor/${FILE_REL}</file>")
+
+        if (FILE_REL STREQUAL "tor" OR FILE_REL STREQUAL "tor.exe")
+            set(TOR_BIN_FOUND 1)
+        endif()
+    endforeach()
+
+    if (NOT TOR_BIN_FOUND)
+        message(FATAL_ERROR "TOR_DIR was specified but the Tor binary could not be found")
+    endif()
+endif()
+
+list(JOIN QRC_LIST "\n" QRC_DATA)
+configure_file("cmake/assets_tor.qrc" "${CMAKE_CURRENT_SOURCE_DIR}/src/assets_tor.qrc")

cmake/assets_tor.qrc

@@ -1,5 +1,5 @@
 <!DOCTYPE RCC><RCC version="1.0">
     <qresource prefix="/">
-        <file>assets/exec/tor</file>
+@QRC_DATA@
     </qresource>
 </RCC>

src/CMakeLists.txt

@@ -25,7 +25,7 @@ find_package(Qt5 REQUIRED COMPONENTS ${QT5_COMPONENTS})
 
 add_subdirectory(openpgp)
 
-qt5_add_resources(RESOURCES assets.qrc)
+qt5_add_resources(RESOURCES assets.qrc assets_tor.qrc)
 
 # Compile source files (.h/.cpp)
 file(GLOB SOURCE_FILES
@@ -70,14 +70,6 @@ if (WITH_SCANNER)
             ${SCANNER_FILES})
 endif()
 
-if(TOR_BIN)
-    if(APPLE)
-        set(ASSETS_TOR "assets_tor_macos.qrc")
-    else()
-        set(ASSETS_TOR "assets_tor.qrc")
-    endif()
-endif()
-
 set(EXECUTABLE_FLAG)
 if(MINGW)
     set(EXECUTABLE_FLAG WIN32)
@@ -164,7 +156,7 @@ if(LOCALMONERO)
     target_compile_definitions(feather PRIVATE HAS_LOCALMONERO=1)
 endif()
 
-if(TOR_BIN)
+if(TOR_DIR)
     target_compile_definitions(feather PRIVATE HAS_TOR_BIN=1)
 endif()
 

src/assets_tor_macos.qrc

@@ -1,6 +0,0 @@
-<!DOCTYPE RCC><RCC version="1.0">
-    <qresource prefix="/">
-        <file>assets/exec/tor</file>
-        <file>assets/exec/libevent-2.1.7.dylib</file>
-    </qresource>
-</RCC>

src/main.cpp

@@ -25,9 +25,7 @@ int main(int argc, char *argv[])
 {
     Q_INIT_RESOURCE(assets);
 
-#if defined(Q_OS_MAC) && defined(HAS_TOR_BIN)
+#if defined(HAS_TOR_BIN)
-    Q_INIT_RESOURCE(assets_tor_macos);
-#elif defined(HAS_TOR_BIN)
     Q_INIT_RESOURCE(assets_tor);
 #endif
 
@@ -194,6 +192,11 @@ if (AttachConsole(ATTACH_PARENT_PROCESS)) {
         else info["Mode"] = "Mainnet";
         info["SSL"] = QSslSocket::sslLibraryVersionString();
         info["SSL build"] = QSslSocket::sslLibraryBuildVersionString();
+#if defined(TOR_VERSION)
+        info["Tor version"] = TOR_VERSION;
+#else
+        info["Tor version"] = "Not bundled";
+#endif
         for (const QString &k: info.keys()) {
             qWarning().nospace().noquote() << QString("%1: %2").arg(k, info[k]);
         }

src/utils/TorManager.cpp

@@ -157,35 +157,13 @@ void TorManager::handleProcessError(QProcess::ProcessError error) {
 }
 
 bool TorManager::unpackBins() {
-    QString torFile;
+    QString torBin = "tor";
-
-    // On MacOS write libevent to disk
-#if defined(Q_OS_MAC)
-    QString libEvent = ":/assets/exec/libevent-2.1.7.dylib";
-    if (Utils::fileExists(libEvent)) {
-        QFile e(libEvent);
-        QFileInfo eventInfo(e);
-        auto libEventPath = QDir(this->torDir).filePath(eventInfo.fileName());
-        qDebug() << libEventPath;
-        e.copy(libEventPath);
-        e.close();
-    }
-#endif
-
-    torFile = ":/assets/exec/tor";
-    if (!Utils::fileExists(torFile))
-        return false;
-
-    // write to disk
-    QFile f(torFile);
-    QFileInfo fileInfo(f);
-    this->torPath = QDir(this->torDir).filePath(fileInfo.fileName());
-
 #if defined(Q_OS_WIN)
-    if(!this->torPath.endsWith(".exe"))
+   torBin += ".exe";
-        this->torPath += ".exe";
 #endif
 
+    this->torPath = QDir(this->torDir).filePath(torBin);
+
     SemanticVersion embeddedVersion = SemanticVersion::fromString(QString(TOR_VERSION));
     SemanticVersion filesystemVersion = this->getVersion(torPath);
     qDebug() << QString("Tor versions: embedded %1, filesystem %2").arg(embeddedVersion.toString(), filesystemVersion.toString());
@@ -194,18 +172,29 @@ bool TorManager::unpackBins() {
         QFile::setPermissions(torPath, QFile::ReadOther | QFile::WriteOther);
         if (!QFile::remove(torPath)) {
             qWarning() << "Unable to remove old Tor binary";
-        };
+            return false;
+        }
     }
 
-    qDebug() << "Writing Tor executable to " << this->torPath;
+    if (embeddedVersion > filesystemVersion) {
-    f.copy(torPath);
+        QDirIterator it(":/assets/tor", QDirIterator::Subdirectories);
-    f.close();
+        while (it.hasNext()) {
+            QString assetFile = it.next();
+            QFileInfo assetFileInfo = QFileInfo(assetFile);
+            QFile f(assetFile);
+            QString filePath = QDir(this->torDir).filePath(assetFileInfo.fileName());
+            f.copy(filePath);
+            f.close();
+        }
+        qInfo() << "Wrote Tor binaries to: " << this->torDir;
+    }
 
 #if defined(Q_OS_UNIX)
-    QFile torBin(this->torPath);
+    QFile tor(this->torPath);
-    torBin.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther
+    tor.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther
-                          | QFile::ReadOwner | QFile::ReadGroup | QFile::ReadOther);
+    | QFile::ReadOwner | QFile::ReadGroup | QFile::ReadOther);
 #endif
+
     return true;
 }