Replace reqwest with simple-request

reqwest was replaced with hyper and hyper-rustls within monero-serai due to reqwest *solely* offering a connection pool API. In the process, it was demonstrated how quickly we can achieve equivalent functionality to reqwest for our use cases with a fraction of the code. This adds our own reqwest alternative to the tree, applying it to both bitcoin-serai and message-queue. By doing so, bitcoin-serai decreases its tree by 21 packages and the processor by 18. Cargo.lock decreases by 8 dependencies, solely adding simple-request. Notably removed is openssl-sys and openssl. One noted decrease functionality is the requirement on the system having installed CA certificates. While we could fallback to the rustls certificates if the system doesn't have any, that's blocked by https://github.com/rustls/hyper-rustls/pulls/228.
2024-12-22 19:49:22 +00:00 · 2023-11-06 09:14:46 -05:00 · 2023-11-06 09:14:46 -05:00 · b9983bf133
commit b9983bf133
parent cddb44ae3f
11 changed files with 204 additions and 119 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -535,10 +535,10 @@ dependencies = [
 "k256",
 "modular-frost",
 "rand_core",
- "reqwest",
 "secp256k1",
 "serde",
 "serde_json",
+ "simple-request",
 "std-shims",
 "thiserror",
 "tokio",
@ -2573,21 +2573,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "fork-tree"
 version = "3.0.0"
@ -3429,19 +3414,6 @@ dependencies = [
 "webpki-roots",
 ]

-[[package]]
-name = "hyper-tls"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
-dependencies = [
- "bytes",
- "hyper",
- "native-tls",
- "tokio",
- "tokio-native-tls",
-]
-
 [[package]]
 name = "hyperlocal"
 version = "0.8.0"
@ -5110,24 +5082,6 @@ dependencies = [
 "rand",
 ]

-[[package]]
-name = "native-tls"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
-dependencies = [
- "lazy_static",
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
-]
-
 [[package]]
 name = "netlink-packet-core"
 version = "0.4.2"
@ -5403,50 +5357,12 @@ dependencies = [
 "syn 1.0.109",
 ]

-[[package]]
-name = "openssl"
-version = "0.10.59"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33"
-dependencies = [
- "bitflags 2.4.1",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.39",
-]
-
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

-[[package]]
-name = "openssl-sys"
-version = "0.9.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "option-ext"
 version = "0.2.0"
@ -6617,12 +6533,10 @@ dependencies = [
 "http",
 "http-body",
 "hyper",
- "hyper-tls",
 "ipnet",
 "js-sys",
 "log",
 "mime",
- "native-tls",
 "once_cell",
 "percent-encoding",
 "pin-project-lite 0.2.13",
@ -6631,7 +6545,6 @@ dependencies = [
 "serde_urlencoded",
 "system-configuration",
 "tokio",
- "tokio-native-tls",
 "tower-service",
 "url",
 "wasm-bindgen",
@ -8403,13 +8316,13 @@ dependencies = [
 "log",
 "once_cell",
 "rand_core",
- "reqwest",
 "schnorr-signatures",
 "serai-db",
 "serai-env",
 "serai-primitives",
 "serde",
 "serde_json",
+ "simple-request",
 "tokio",
 "zeroize",
 ]
@ -8858,6 +8771,17 @@ dependencies = [
 "wide",
 ]

+[[package]]
+name = "simple-request"
+version = "0.1.0"
+dependencies = [
+ "base64ct",
+ "hyper",
+ "hyper-rustls",
+ "tokio",
+ "zeroize",
+]
+
 [[package]]
 name = "simple_asn1"
 version = "0.6.2"
@ -10162,16 +10086,6 @@ dependencies = [
 "syn 2.0.39",
 ]

-[[package]]
-name = "tokio-native-tls"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
-dependencies = [
- "native-tls",
- "tokio",
-]
-
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -5,6 +5,7 @@ members = [
  "common/zalloc",
  "common/db",
  "common/env",
+  "common/request",

  "crypto/transcript",

--- a/coins/bitcoin/Cargo.toml
+++ b/coins/bitcoin/Cargo.toml
@ -25,7 +25,7 @@ frost = { package = "modular-frost", path = "../../crypto/frost", version = "0.8
 hex = { version = "0.4", default-features = false, optional = true }
 serde = { version = "1", default-features = false, features = ["derive"], optional = true }
 serde_json = { version = "1", default-features = false, optional = true }
-reqwest = { version = "0.11", default-features = false, features = ["default-tls", "json"], optional = true }
+simple-request = { path = "../../common/request", version = "0.1", default-features = false, features = ["basic-auth"], optional = true }

 [dev-dependencies]
 secp256k1 = { version = "0.28", default-features = false, features = ["std"] }
@ -54,7 +54,7 @@ std = [
  "hex/std",
  "serde/std",
  "serde_json/std",
-  "reqwest",
+  "simple-request",
 ]
 hazmat = []
 default = ["std"]
--- a/coins/bitcoin/src/rpc.rs
+++ b/coins/bitcoin/src/rpc.rs
@ -6,7 +6,7 @@ use thiserror::Error;
 use serde::{Deserialize, de::DeserializeOwned};
 use serde_json::json;

-use reqwest::Client;
+use simple_request::{Request, Client};

 use bitcoin::{
  hashes::{Hash, hex::FromHex},
@ -62,7 +62,7 @@ impl Rpc {
  /// provided to this library, if the RPC has an incompatible argument layout. That is not checked
  /// at time of RPC creation.
  pub async fn new(url: String) -> Result<Rpc, RpcError> {
-    let rpc = Rpc { client: Client::new(), url };
+    let rpc = Rpc { client: Client::with_connection_pool(), url };

    // Make an RPC request to verify the node is reachable and sane
    let res: String = rpc.rpc_call("help", json!([])).await?;
@ -107,19 +107,26 @@ impl Rpc {
    method: &str,
    params: serde_json::Value,
  ) -> Result<Response, RpcError> {
-    let res = self
+    let mut res = self
      .client
-      .post(&self.url)
-      .json(&json!({ "jsonrpc": "2.0", "method": method, "params": params }))
-      .send()
+      .request(
+        Request::post(&self.url)
+          .header("Content-Type", "application/json")
+          .body(
+            serde_json::to_vec(&json!({ "jsonrpc": "2.0", "method": method, "params": params }))
+              .unwrap()
+              .into(),
+          )
+          .unwrap(),
+      )
      .await
      .map_err(|_| RpcError::ConnectionError)?
-      .text()
+      .body()
      .await
      .map_err(|_| RpcError::ConnectionError)?;

    let res: RpcResponse<Response> =
-      serde_json::from_str(&res).map_err(|e| RpcError::InvalidJson(e.classify()))?;
+      serde_json::from_reader(&mut res).map_err(|e| RpcError::InvalidJson(e.classify()))?;
    match res {
      RpcResponse::Ok { result } => Ok(result),
      RpcResponse::Err { error } => Err(RpcError::RequestError(error)),
--- a/coins/monero/Cargo.toml
+++ b/coins/monero/Cargo.toml
@ -55,6 +55,7 @@ base58-monero = { version = "2", default-features = false, features = ["check"]

 # Used for the provided HTTP RPC
 digest_auth = { version = "0.3", default-features = false, optional = true }
+# Deprecated here means to enable deprecated warnings, not to restore deprecated APIs
 hyper = { version = "0.14", default-features = false, features = ["http1", "tcp", "client", "backports", "deprecated"], optional = true }
 hyper-rustls = { version = "0.24", default-features = false, features = ["http1", "native-tokio"], optional = true }
 tokio = { version = "1", default-features = false, optional = true }
--- a/common/request/Cargo.toml
+++ b/common/request/Cargo.toml
@ -0,0 +1,25 @@
+[package]
+name = "simple-request"
+version = "0.1.0"
+description = "A simple HTTP(S) request library"
+license = "MIT"
+repository = "https://github.com/serai-dex/serai/tree/develop/common/simple-request"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = ["http", "https", "async", "request", "ssl"]
+edition = "2021"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[dependencies]
+# Deprecated here means to enable deprecated warnings, not to restore deprecated APIs
+hyper = { version = "0.14", default-features = false, features = ["http1", "tcp", "client", "backports", "deprecated"] }
+hyper-rustls = { version = "0.24", default-features = false, features = ["http1", "native-tokio"] }
+tokio = { version = "1", default-features = false }
+
+zeroize = { version = "1", optional = true }
+base64ct = { version = "1", features = ["alloc"], optional = true }
+
+[features]
+basic-auth = ["zeroize", "base64ct"]
--- a/common/request/LICENSE
+++ b/common/request/LICENSE
@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Luke Parker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/common/request/README.md
+++ b/common/request/README.md
@ -0,0 +1,7 @@
+# Simple Request
+
+A simple alternative to reqwest, supporting HTTPS, intended to support a
+majority of use cases with a fraction of the dependency tree.
+
+This library is built directly around `hyper`, `hyper-rustls`, and does require
+`tokio`. Support for `async-std` would be welcome.
--- a/common/request/src/lib.rs
+++ b/common/request/src/lib.rs
@ -0,0 +1,97 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![doc = include_str!("../README.md")]
+
+use hyper_rustls::{HttpsConnectorBuilder, HttpsConnector};
+use hyper::{
+  StatusCode,
+  header::{HeaderValue, HeaderMap},
+  body::{Buf, Body},
+  Response as HyperResponse,
+  client::HttpConnector,
+};
+pub use hyper::{self, Request};
+
+#[derive(Debug)]
+pub struct Response(HyperResponse<Body>);
+impl Response {
+  pub fn status(&self) -> StatusCode {
+    self.0.status()
+  }
+  pub fn headers(&self) -> &HeaderMap<HeaderValue> {
+    self.0.headers()
+  }
+  pub async fn body(self) -> Result<impl std::io::Read, hyper::Error> {
+    Ok(hyper::body::aggregate(self.0.into_body()).await?.reader())
+  }
+}
+
+#[derive(Clone, Debug)]
+enum Connection {
+  ConnectionPool(hyper::Client<HttpsConnector<HttpConnector>>),
+}
+
+#[derive(Clone, Debug)]
+pub struct Client {
+  connection: Connection,
+}
+
+#[derive(Debug)]
+pub enum Error {
+  InvalidHost,
+  Hyper(hyper::Error),
+}
+
+impl Client {
+  fn https_builder() -> HttpsConnector<HttpConnector> {
+    HttpsConnectorBuilder::new().with_native_roots().https_or_http().enable_http1().build()
+  }
+
+  pub fn with_connection_pool() -> Client {
+    Client {
+      connection: Connection::ConnectionPool(hyper::Client::builder().build(Self::https_builder())),
+    }
+  }
+
+  /*
+  fn without_connection_pool() -> Client {}
+  */
+
+  pub async fn request(&self, mut request: Request<Body>) -> Result<Response, Error> {
+    if request.headers().get(hyper::header::HOST).is_none() {
+      let host = request.uri().host().ok_or(Error::InvalidHost)?.to_string();
+      request
+        .headers_mut()
+        .insert(hyper::header::HOST, HeaderValue::from_str(&host).map_err(|_| Error::InvalidHost)?);
+    }
+
+    #[cfg(feature = "basic-auth")]
+    if request.headers().get(hyper::header::AUTHORIZATION).is_none() {
+      if let Some(authority) = request.uri().authority() {
+        let authority = authority.as_str();
+        if authority.contains('@') {
+          // Decode the username and password from the URI
+          let mut userpass = authority.split('@').next().unwrap().to_string();
+          // If the password is "", the URI may omit :, yet the authentication will still expect it
+          if !userpass.contains(':') {
+            userpass.push(':');
+          }
+
+          use zeroize::Zeroize;
+          use base64ct::{Encoding, Base64};
+
+          let mut encoded = Base64::encode_string(userpass.as_bytes());
+          userpass.zeroize();
+          request.headers_mut().insert(
+            hyper::header::AUTHORIZATION,
+            HeaderValue::from_str(&format!("Basic {encoded}")).unwrap(),
+          );
+          encoded.zeroize();
+        }
+      }
+    }
+
+    Ok(Response(match &self.connection {
+      Connection::ConnectionPool(client) => client.request(request).await.map_err(Error::Hyper)?,
+    }))
+  }
+}
--- a/message-queue/Cargo.toml
+++ b/message-queue/Cargo.toml
@ -46,7 +46,7 @@ serai-env = { path = "../common/env" }
 serai-primitives = { path = "../substrate/primitives" }

 jsonrpsee = { version = "0.16", default-features = false, features = ["server"], optional = true }
-reqwest = { version = "0.11", default-features = false, features = ["json"] }
+simple-request = { path = "../common/request", default-features = false }

 [features]
 binaries = ["serai-db", "serai-db/rocksdb", "jsonrpsee"]
--- a/message-queue/src/client.rs
+++ b/message-queue/src/client.rs
@ -11,7 +11,7 @@ use schnorr_signatures::SchnorrSignature;

 use serde::{Serialize, Deserialize};

-use reqwest::Client;
+use simple_request::{Request, Client};

 use serai_env as env;

@ -45,7 +45,7 @@ impl MessageQueue {
      service,
      pub_key: Ristretto::generator() * priv_key.deref(),
      priv_key,
-      client: Client::new(),
+      client: Client::with_connection_pool(),
      url,
    }
  }
@ -81,18 +81,30 @@ impl MessageQueue {
      id: u64,
    }

-    let res = loop {
+    let mut res = loop {
      // Make the request
      match self
        .client
-        .post(&self.url)
-        .json(&JsonRpcRequest { jsonrpc: "2.0", method, params: params.clone(), id: 0 })
-        .send()
+        .request(
+          Request::post(&self.url)
+            .header("Content-Type", "application/json")
+            .body(
+              serde_json::to_vec(&JsonRpcRequest {
+                jsonrpc: "2.0",
+                method,
+                params: params.clone(),
+                id: 0,
+              })
+              .unwrap()
+              .into(),
+            )
+            .unwrap(),
+        )
        .await
      {
        Ok(req) => {
          // Get the response
-          match req.text().await {
+          match req.body().await {
            Ok(res) => break res,
            Err(e) => {
              dbg!(e);
@ -108,8 +120,8 @@ impl MessageQueue {
      tokio::time::sleep(core::time::Duration::from_secs(1)).await;
    };

-    let json =
-      serde_json::from_str::<serde_json::Value>(&res).expect("message-queue returned invalid JSON");
+    let json: serde_json::Value =
+      serde_json::from_reader(&mut res).expect("message-queue returned invalid JSON");
    if json.get("result").is_none() {
      panic!("call failed: {json}");
    }