download monerod in CI (#123)

* download monerod in CI

* move action file

* add macOS arm support

* remove reqwest from workspace

* undo whitespace changes

* fix indentation

* update comments

* add monerod to .gitignore

* Apply suggestions from code review

Co-authored-by: hinto-janai <hinto.janai@protonmail.com>

* add back spacing

---------

Co-authored-by: hinto-janai <hinto.janai@protonmail.com>

.github/actions/monerod-download/action.yml

@@ -0,0 +1,62 @@
+#  MIT License
+#
+#  Copyright (c) 2022-2023 Luke Parker
+#  Copyright (c) Cuprate developers
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#
+#  Initially taken from Serai Dex: https://github.com/serai-dex/serai/blob/b823413c9b7ae6747b9af99e18379cfc49f4271a/.github/actions/monero/action.yml.
+
+name: monerod-download
+description: Downloads the core Monero daemon
+
+inputs:
+  version:
+    description: "Version to download"
+    required: false
+    default: v0.18.3.3
+
+runs:
+  using: "composite"
+  steps:
+    - name: Monero Daemon Cache
+      id: cache-monerod
+      uses: actions/cache@v3
+      with:
+        path: |
+          monerod
+          monerod.exe
+        key: monerod-${{ runner.os }}-${{ runner.arch }}-${{ inputs.version }}
+
+    - name: Download the Monero Daemon
+      if: steps.cache-monerod.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        OS=${{ runner.os }}
+        ARCH=${{ runner.arch }}
+
+        case "$OS $ARCH" in
+          "Windows X64") FILE=monero-win-x64-${{ inputs.version }}.zip ;;
+          "Windows X86") FILE=monero-win-x86-${{ inputs.version }}.zip ;;
+          "Linux X64") FILE=monero-linux-x64-${{ inputs.version }}.tar.bz2 ;;
+          "Linux X86") FILE=monero-linux-x86-${{ inputs.version }}.tar.bz2 ;;
+          "macOS X64") FILE=monero-mac-x64-${{ inputs.version }}.tar.bz2 ;;
+          "macOS ARM64") FILE=monero-mac-armv8-${{ inputs.version }}.tar.bz2 ;;
+          *) exit 1 ;;
+        esac
+        curl -O -L https://downloads.getmonero.org/cli/$FILE
+        if [[ ${{ runner.os }} == Windows ]]; then
+          unzip $FILE
+          mv */monerod.exe monerod.exe
+        else
+          tar -xvf $FILE
+          mv */monerod monerod
+        fi

.github/workflows/ci.yml

@@ -77,6 +77,9 @@ jobs:
           ~/.rustup
         key: ${{ matrix.os }}
 
+    - name: Download monerod
+      uses: ./.github/actions/monerod-download
+
     # Packages other than `Boost` used by `Monero` are listed here.
     # https://github.com/monero-project/monero/blob/c444a7e002036e834bfb4c68f04a121ce1af5825/.github/workflows/build.yml#L71
 
@@ -103,7 +106,7 @@ jobs:
         rustup default stable-x86_64-pc-windows-gnu
 
     - name: Documentation
-      run: cargo doc --workspace --all-features
+      run: cargo doc --workspace --all-features --no-deps
 
     - name: Clippy (fail on warnings)
       run: cargo clippy --workspace --all-features --all-targets -- -D warnings
@@ -113,7 +116,7 @@ jobs:
       run: |
         cargo test --all-features --workspace
         cargo test --package cuprate-database --no-default-features --features redb --features service
-
+        
     # TODO: upload binaries with `actions/upload-artifact@v3`
     - name: Build
       run: cargo build --all-features --all-targets --workspace

.gitignore

@@ -1,2 +1,3 @@
 target/
 .vscode
+monerod

Cargo.toml

@@ -75,7 +75,6 @@ tracing               = { version = "0.1.40", default-features = false }
 
 ## workspace.dev-dependencies
 tempfile              = { version = "3" }
-reqwest               = { version = "0.11.24" }
 pretty_assertions     = { version = "1.4.0" }
 proptest              = { version = "1" }
 proptest-derive       = { version = "0.4.0" }

test-utils/Cargo.toml

@@ -18,7 +18,6 @@ futures = { workspace = true, features = ["std"] }
 async-trait = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-util = { workspace = true }
-reqwest = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 bytes = { workspace = true, features = ["std"] }
@@ -26,13 +25,6 @@ tempfile = { workspace = true }
 
 borsh = { workspace = true, features = ["derive"]}
 
-[target.'cfg(unix)'.dependencies]
-tar = "0.4.40"
-bzip2 = "0.4.4"
-
-[target.'cfg(windows)'.dependencies]
-zip = "0.6"
-
 [dev-dependencies]
 hex = { workspace = true }
 pretty_assertions = { workspace = true }

test-utils/src/monerod.rs

@@ -4,9 +4,12 @@
 //! this to test compatibility with monerod.
 //!
 use std::{
+    env::current_dir,
     ffi::OsStr,
+    fs::read_dir,
     io::Read,
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener},
+    path::PathBuf,
     process::{Child, Command, Stdio},
     str::from_utf8,
     thread::panicking,
@@ -15,14 +18,9 @@ use std::{
 
 use tokio::{task::yield_now, time::timeout};
 
-mod download;
-
 /// IPv4 local host.
 const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
 
-/// The `monerod` version to use.
-const MONEROD_VERSION: &str = "v0.18.3.1";
-
 /// The log line `monerod` emits indicated it has successfully started up.
 const MONEROD_STARTUP_TEXT: &str =
     "The daemon will start synchronizing with the network. This may take a long time to complete.";
@@ -34,7 +32,7 @@ const MONEROD_SHUTDOWN_TEXT: &str = "Stopping cryptonote protocol";
 ///
 /// This function will set `regtest` and the P2P/ RPC ports so these can't be included in the flags.
 pub async fn monerod<T: AsRef<OsStr>>(flags: impl IntoIterator<Item = T>) -> SpawnedMoneroD {
-    let path_to_monerod = download::check_download_monerod().await.unwrap();
+    let path_to_monerod = find_root().join("monerod");
 
     let rpc_port = get_available_port(&[]);
     let p2p_port = get_available_port(&[rpc_port]);
@@ -54,7 +52,9 @@ pub async fn monerod<T: AsRef<OsStr>>(flags: impl IntoIterator<Item = T>) -> Spa
         .arg(format!("--data-dir={}", data_dir.path().display()))
         .arg("--non-interactive")
         .spawn()
-        .unwrap();
+        .expect(
+            "Failed to start monerod, you need to have the monerod binary in the root of the repo",
+        );
 
     let mut logs = String::new();
 
@@ -92,6 +92,20 @@ pub async fn monerod<T: AsRef<OsStr>>(flags: impl IntoIterator<Item = T>) -> Spa
     }
 }
 
+/// Finds the root of the repo by finding the `target` directory, this will work up from the current
+/// directory until it finds a `target` directory, then returns the directory that the target is contained
+/// in.
+fn find_root() -> PathBuf {
+    let mut current_dir = current_dir().unwrap();
+    loop {
+        if read_dir(current_dir.join("target")).is_ok() {
+            return current_dir;
+        } else if !current_dir.pop() {
+            panic!("Could not find ./target");
+        }
+    }
+}
+
 /// Fetch an available TCP port on the machine for `monerod` to bind to.
 fn get_available_port(already_taken: &[u16]) -> u16 {
     loop {

test-utils/src/monerod/download.rs

@@ -1,106 +0,0 @@
-//! Downloading Monerod Module
-//!
-//! This module handles finding the right monerod file to download, downloading it and extracting it.
-//!
-use std::{
-    env::{
-        consts::{ARCH, OS},
-        current_dir,
-    },
-    fs::read_dir,
-    path::{Path, PathBuf},
-};
-
-#[cfg(unix)]
-use bytes::Buf;
-use reqwest::{get, Error as ReqError};
-use tokio::sync::Mutex;
-
-use super::MONEROD_VERSION;
-
-/// A mutex to make sure only one thread at a time downloads monerod.
-static DOWNLOAD_MONEROD_MUTEX: Mutex<()> = Mutex::const_new(());
-
-/// Returns the file name to download and the expected extracted folder name.
-fn file_name(version: &str) -> (String, String) {
-    let download_file = match (OS, ARCH) {
-        ("windows", "x64" | "x86_64") => format!("monero-win-x64-{version}.zip"),
-        ("windows", "x86") => format!("monero-win-x86-{version}.zip"),
-        ("linux", "x64" | "x86_64") => format!("monero-linux-x64-{version}.tar.bz2"),
-        ("linux", "x86") => format!("monero-linux-x86-{version}.tar.bz2"),
-        ("macos", "x64" | "x86_64") => format!("monero-mac-x64-{version}.tar.bz2"),
-        ("macos", "aarch64") => format!("monero-mac-armv8-{version}.tar.bz2"),
-        _ => panic!("Can't get monerod for {OS}, {ARCH}."),
-    };
-
-    let extracted_dir = match (OS, ARCH) {
-        ("windows", "x64" | "x86_64") => {
-            format!("monero-x86_64-w64-mingw32-{version}")
-        }
-        ("windows", "x86") => format!("monero-i686-w64-mingw32-{version}"),
-        ("linux", "x64" | "x86_64") => format!("monero-x86_64-linux-gnu-{version}"),
-        ("linux", "x86") => format!("monero-i686-linux-gnu-{version}"),
-        ("macos", "x64" | "x86_64") => {
-            format!("monero-x86_64-apple-darwin11-{version}")
-        }
-        ("macos", "aarch64") => format!("monero-aarch64-apple-darwin11-{version}"),
-        _ => panic!("Can't get monerod for {OS}, {ARCH}."),
-    };
-
-    (download_file, extracted_dir)
-}
-
-/// Downloads the monerod file provided, extracts it and puts the extracted folder into `path_to_store`.
-async fn download_monerod(file_name: &str, path_to_store: &Path) -> Result<(), ReqError> {
-    let res = get(format!("https://downloads.getmonero.org/cli/{file_name}")).await?;
-    let monerod_archive = res.bytes().await.unwrap();
-
-    #[cfg(unix)]
-    {
-        let bzip_decomp = bzip2::read::BzDecoder::new(monerod_archive.reader());
-        let mut tar_archive = tar::Archive::new(bzip_decomp);
-        tar_archive.unpack(path_to_store).unwrap();
-    }
-    #[cfg(windows)]
-    {
-        let mut zip = zip::ZipArchive::new(std::io::Cursor::new(monerod_archive.as_ref())).unwrap();
-        zip.extract(path_to_store).unwrap();
-    }
-
-    Ok(())
-}
-
-/// Finds the `target` directory, this will work up from the current directory until
-/// it finds a `target` directory.
-fn find_target() -> PathBuf {
-    let mut current_dir = current_dir().unwrap();
-    loop {
-        let potential_target = current_dir.join("target");
-        if read_dir(current_dir.join("target")).is_ok() {
-            return potential_target;
-        } else if !current_dir.pop() {
-            panic!("Could not find ./target");
-        }
-    }
-}
-
-/// Checks if we have monerod or downloads it if we don't and then returns the path to it.
-pub(crate) async fn check_download_monerod() -> Result<PathBuf, ReqError> {
-    // make sure no other threads are downloading monerod at the same time.
-    let _guard = DOWNLOAD_MONEROD_MUTEX.lock().await;
-
-    let path_to_store = find_target();
-
-    let (file_name, dir_name) = file_name(MONEROD_VERSION);
-
-    let path_to_monerod = path_to_store.join(dir_name);
-
-    // Check if we already have monerod
-    if read_dir(&path_to_monerod).is_ok() {
-        return Ok(path_to_monerod.join("monerod"));
-    }
-
-    download_monerod(&file_name, &path_to_store).await?;
-
-    Ok(path_to_monerod.join("monerod"))
-}