From 2edc2f36123ab8bcd972f4392651f3e04cbd7fce Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Fri, 13 Sep 2024 23:51:53 -0400 Subject: [PATCH] Add a database of all Monero outs into the processor Enables synchronous transaction creation (which requires synchronous decoy selection). --- Cargo.lock | 1 + networks/monero/rpc/src/lib.rs | 6 +- processor/monero/Cargo.toml | 1 + processor/monero/src/decoys.rs | 294 ++++++++++++++++++++++++++++++ processor/monero/src/lib.rs | 140 -------------- processor/monero/src/main.rs | 2 + processor/monero/src/rpc.rs | 27 +-- processor/monero/src/scheduler.rs | 142 +++++++++++++++ 8 files changed, 457 insertions(+), 156 deletions(-) create mode 100644 processor/monero/src/decoys.rs diff --git a/Cargo.lock b/Cargo.lock index b08cde03..9e34ea3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8515,6 +8515,7 @@ version = "0.1.0" dependencies = [ "borsh", "ciphersuite", + "curve25519-dalek", "dalek-ff-group", "dkg", "flexible-transcript", diff --git a/networks/monero/rpc/src/lib.rs b/networks/monero/rpc/src/lib.rs index 4c5055cc..3c8d337a 100644 --- a/networks/monero/rpc/src/lib.rs +++ b/networks/monero/rpc/src/lib.rs @@ -249,7 +249,7 @@ fn rpc_point(point: &str) -> Result { /// While no implementors are directly provided, [monero-simple-request-rpc]( /// https://github.com/serai-dex/serai/tree/develop/networks/monero/rpc/simple-request /// ) is recommended. -pub trait Rpc: Sync + Clone + Debug { +pub trait Rpc: Sync + Clone { /// Perform a POST request to the specified route with the specified body. /// /// The implementor is left to handle anything such as authentication. @@ -1003,10 +1003,10 @@ pub trait Rpc: Sync + Clone + Debug { /// An implementation is provided for any satisfier of `Rpc`. It is not recommended to use an `Rpc` /// object to satisfy this. This should be satisfied by a local store of the output distribution, /// both for performance and to prevent potential attacks a remote node can perform. -pub trait DecoyRpc: Sync + Clone + Debug { +pub trait DecoyRpc: Sync { /// Get the height the output distribution ends at. /// - /// This is equivalent to the hight of the blockchain it's for. This is intended to be cheaper + /// This is equivalent to the height of the blockchain it's for. This is intended to be cheaper /// than fetching the entire output distribution. fn get_output_distribution_end_height( &self, diff --git a/processor/monero/Cargo.toml b/processor/monero/Cargo.toml index 6f9ce40a..436f327e 100644 --- a/processor/monero/Cargo.toml +++ b/processor/monero/Cargo.toml @@ -25,6 +25,7 @@ scale = { package = "parity-scale-codec", version = "3", default-features = fals borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] } transcript = { package = "flexible-transcript", path = "../../crypto/transcript", default-features = false, features = ["std", "recommended"] } +curve25519-dalek = { version = "4", default-features = false, features = ["alloc", "zeroize"] } dalek-ff-group = { path = "../../crypto/dalek-ff-group", default-features = false, features = ["std"] } ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["std", "ed25519"] } dkg = { path = "../../crypto/dkg", default-features = false, features = ["std", "evrf-ed25519"] } diff --git a/processor/monero/src/decoys.rs b/processor/monero/src/decoys.rs new file mode 100644 index 00000000..000463d0 --- /dev/null +++ b/processor/monero/src/decoys.rs @@ -0,0 +1,294 @@ +use core::{ + future::Future, + ops::{Bound, RangeBounds}, +}; + +use curve25519_dalek::{ + scalar::Scalar, + edwards::{CompressedEdwardsY, EdwardsPoint}, +}; +use monero_wallet::{ + DEFAULT_LOCK_WINDOW, + primitives::Commitment, + transaction::{Timelock, Input, Pruned, Transaction}, + rpc::{OutputInformation, RpcError, Rpc as MRpcTrait, DecoyRpc}, +}; + +use borsh::{BorshSerialize, BorshDeserialize}; +use serai_db::{Get, DbTxn, Db, create_db}; + +use primitives::task::ContinuallyRan; +use scanner::ScannerFeed; + +use crate::Rpc; + +#[derive(BorshSerialize, BorshDeserialize)] +struct EncodableOutputInformation { + height: u64, + timelocked: bool, + key: [u8; 32], + commitment: [u8; 32], +} + +create_db! { + MoneroProcessorDecoys { + NextToIndexBlock: () -> u64, + PriorIndexedBlock: () -> [u8; 32], + DistributionStartBlock: () -> u64, + Distribution: () -> Vec, + Out: (index: u64) -> EncodableOutputInformation, + } +} + +/* + We want to be able to select decoys when planning transactions, but planning transactions is a + synchronous process. We store the decoys to a local database and have our database implement + `DecoyRpc` to achieve synchronous decoy selection. + + This is only needed as the transactions we sign must have decoys decided and agreed upon. With + FCMP++s, we'll be able to sign transactions without the membership proof, letting any signer + prove for membership after the fact (with their local views). Until then, this task remains. +*/ +pub(crate) struct DecoysTask { + pub(crate) rpc: Rpc, + pub(crate) current_distribution: Vec, +} + +impl ContinuallyRan for DecoysTask { + fn run_iteration(&mut self) -> impl Send + Future> { + async move { + let finalized_block_number = self + .rpc + .rpc + .get_height() + .await + .map_err(|e| format!("couldn't fetch latest block number: {e:?}"))? + .checked_sub(Rpc::::CONFIRMATIONS.try_into().unwrap()) + .ok_or(format!( + "blockchain only just started and doesn't have {} blocks yet", + Rpc::::CONFIRMATIONS + ))?; + + if NextToIndexBlock::get(&self.rpc.db).is_none() { + let distribution = self + .rpc + .rpc + .get_output_distribution(..= finalized_block_number) + .await + .map_err(|e| format!("failed to get output distribution: {e:?}"))?; + if distribution.is_empty() { + Err("distribution was empty".to_string())?; + } + + let distribution_start_block = finalized_block_number - (distribution.len() - 1); + // There may have been a reorg between the time of getting the distribution and the time of + // getting this block. This is an invariant and assumed not to have happened in the split + // second it's possible. + let block = self + .rpc + .rpc + .get_block_by_number(distribution_start_block) + .await + .map_err(|e| format!("failed to get the start block for the distribution: {e:?}"))?; + + let mut txn = self.rpc.db.txn(); + NextToIndexBlock::set(&mut txn, &distribution_start_block.try_into().unwrap()); + PriorIndexedBlock::set(&mut txn, &block.header.previous); + DistributionStartBlock::set(&mut txn, &u64::try_from(distribution_start_block).unwrap()); + txn.commit(); + } + + let next_to_index_block = + usize::try_from(NextToIndexBlock::get(&self.rpc.db).unwrap()).unwrap(); + if next_to_index_block >= finalized_block_number { + return Ok(false); + } + + for b in next_to_index_block ..= finalized_block_number { + // Fetch the block + let block = self + .rpc + .rpc + .get_block_by_number(b) + .await + .map_err(|e| format!("decoys task failed to fetch block: {e:?}"))?; + let prior = PriorIndexedBlock::get(&self.rpc.db).unwrap(); + if block.header.previous != prior { + panic!( + "decoys task detected reorg: expected {}, found {}", + hex::encode(prior), + hex::encode(block.header.previous) + ); + } + + // Fetch the transactions in the block + let transactions = self + .rpc + .rpc + .get_pruned_transactions(&block.transactions) + .await + .map_err(|e| format!("failed to get the pruned transactions within a block: {e:?}"))?; + + fn outputs( + list: &mut Vec, + block_number: u64, + tx: Transaction, + ) { + match tx { + Transaction::V1 { .. } => {} + Transaction::V2 { prefix, proofs } => { + for (i, output) in prefix.outputs.into_iter().enumerate() { + list.push(EncodableOutputInformation { + // This is correct per the documentation on OutputInformation, which this maps to + height: block_number, + timelocked: prefix.additional_timelock != Timelock::None, + key: output.key.to_bytes(), + commitment: if matches!( + prefix.inputs.first().expect("Monero transaction had no inputs"), + Input::Gen(_) + ) { + Commitment::new( + Scalar::ONE, + output.amount.expect("miner transaction outputs didn't have amounts set"), + ) + .calculate() + .compress() + .to_bytes() + } else { + proofs + .as_ref() + .expect("non-miner V2 transaction didn't have proofs") + .base + .commitments + .get(i) + .expect("amount of commitments didn't match amount of outputs") + .compress() + .to_bytes() + }, + }); + } + } + } + } + + let block_hash = block.hash(); + + let b = u64::try_from(b).unwrap(); + let mut encodable = Vec::with_capacity(2 * (1 + block.transactions.len())); + outputs(&mut encodable, b, block.miner_transaction.into()); + for transaction in transactions { + outputs(&mut encodable, b, transaction); + } + + let existing_outputs = self.current_distribution.last().copied().unwrap_or(0); + let now_outputs = existing_outputs + u64::try_from(encodable.len()).unwrap(); + self.current_distribution.push(now_outputs); + + let mut txn = self.rpc.db.txn(); + NextToIndexBlock::set(&mut txn, &(b + 1)); + PriorIndexedBlock::set(&mut txn, &block_hash); + // TODO: Don't write the entire 10 MB distribution to the DB every two minutes + Distribution::set(&mut txn, &self.current_distribution); + for (b, out) in (existing_outputs .. now_outputs).zip(encodable) { + Out::set(&mut txn, b, &out); + } + txn.commit(); + } + Ok(true) + } + } +} + +// TODO: Cache the distribution in a static +pub(crate) struct Decoys<'a, G: Get>(&'a G); +impl<'a, G: Sync + Get> DecoyRpc for Decoys<'a, G> { + #[rustfmt::skip] + fn get_output_distribution_end_height( + &self, + ) -> impl Send + Future> { + async move { + Ok(NextToIndexBlock::get(self.0).map_or(0, |b| usize::try_from(b).unwrap() + 1)) + } + } + fn get_output_distribution( + &self, + range: impl Send + RangeBounds, + ) -> impl Send + Future, RpcError>> { + async move { + let from = match range.start_bound() { + Bound::Included(from) => *from, + Bound::Excluded(from) => from.checked_add(1).ok_or_else(|| { + RpcError::InternalError("range's from wasn't representable".to_string()) + })?, + Bound::Unbounded => 0, + }; + let to = match range.end_bound() { + Bound::Included(to) => *to, + Bound::Excluded(to) => to + .checked_sub(1) + .ok_or_else(|| RpcError::InternalError("range's to wasn't representable".to_string()))?, + Bound::Unbounded => { + panic!("requested distribution till latest block, which is non-deterministic") + } + }; + if from > to { + Err(RpcError::InternalError(format!( + "malformed range: inclusive start {from}, inclusive end {to}" + )))?; + } + + let distribution_start_block = usize::try_from( + DistributionStartBlock::get(self.0).expect("never populated the distribution start block"), + ) + .unwrap(); + let len_of_distribution_until_to = + to.checked_sub(distribution_start_block).ok_or_else(|| { + RpcError::InternalError( + "requested distribution until a block when the distribution had yet to start" + .to_string(), + ) + })? + + 1; + let distribution = Distribution::get(self.0).expect("never populated the distribution"); + assert!( + distribution.len() >= len_of_distribution_until_to, + "requested distribution until block we have yet to index" + ); + Ok( + distribution[from.saturating_sub(distribution_start_block) .. len_of_distribution_until_to] + .to_vec(), + ) + } + } + fn get_outs( + &self, + _indexes: &[u64], + ) -> impl Send + Future, RpcError>> { + async move { unimplemented!("get_outs is unused") } + } + fn get_unlocked_outputs( + &self, + indexes: &[u64], + height: usize, + fingerprintable_deterministic: bool, + ) -> impl Send + Future>, RpcError>> { + assert!(fingerprintable_deterministic, "processor wasn't using deterministic output selection"); + async move { + let mut res = vec![]; + for index in indexes { + let out = Out::get(self.0, *index).expect("requested output we didn't index"); + let unlocked = (!out.timelocked) && + ((usize::try_from(out.height).unwrap() + DEFAULT_LOCK_WINDOW) <= height); + res.push(unlocked.then(|| CompressedEdwardsY(out.key).decompress()).flatten().map(|key| { + [ + key, + CompressedEdwardsY(out.commitment) + .decompress() + .expect("output with invalid commitment"), + ] + })); + } + Ok(res) + } + } +} diff --git a/processor/monero/src/lib.rs b/processor/monero/src/lib.rs index 1cde1414..52ebb6cb 100644 --- a/processor/monero/src/lib.rs +++ b/processor/monero/src/lib.rs @@ -107,146 +107,6 @@ impl Monero { Ok(FeeRate::new(fee.max(MINIMUM_FEE), 10000).unwrap()) } - async fn make_signable_transaction( - &self, - block_number: usize, - plan_id: &[u8; 32], - inputs: &[Output], - payments: &[Payment], - change: &Option
, - calculating_fee: bool, - ) -> Result, NetworkError> { - for payment in payments { - assert_eq!(payment.balance.coin, Coin::Monero); - } - - // TODO2: Use an fee representative of several blocks, cached inside Self - let block_for_fee = self.get_block(block_number).await?; - let fee_rate = self.median_fee(&block_for_fee).await?; - - // Determine the RCT proofs to make based off the hard fork - // TODO: Make a fn for this block which is duplicated with tests - let rct_type = match block_for_fee.header.hardfork_version { - 14 => RctType::ClsagBulletproof, - 15 | 16 => RctType::ClsagBulletproofPlus, - _ => panic!("Monero hard forked and the processor wasn't updated for it"), - }; - - let mut transcript = - RecommendedTranscript::new(b"Serai Processor Monero Transaction Transcript"); - transcript.append_message(b"plan", plan_id); - - // All signers need to select the same decoys - // All signers use the same height and a seeded RNG to make sure they do so. - let mut inputs_actual = Vec::with_capacity(inputs.len()); - for input in inputs { - inputs_actual.push( - OutputWithDecoys::fingerprintable_deterministic_new( - &mut ChaCha20Rng::from_seed(transcript.rng_seed(b"decoys")), - &self.rpc, - // TODO: Have Decoys take RctType - match rct_type { - RctType::ClsagBulletproof => 11, - RctType::ClsagBulletproofPlus => 16, - _ => panic!("selecting decoys for an unsupported RctType"), - }, - block_number + 1, - input.0.clone(), - ) - .await - .map_err(map_rpc_err)?, - ); - } - - // Monero requires at least two outputs - // If we only have one output planned, add a dummy payment - let mut payments = payments.to_vec(); - let outputs = payments.len() + usize::from(u8::from(change.is_some())); - if outputs == 0 { - return Ok(None); - } else if outputs == 1 { - payments.push(Payment { - address: Address::new( - ViewPair::new(EdwardsPoint::generator().0, Zeroizing::new(Scalar::ONE.0)) - .unwrap() - .legacy_address(MoneroNetwork::Mainnet), - ) - .unwrap(), - balance: Balance { coin: Coin::Monero, amount: Amount(0) }, - data: None, - }); - } - - let payments = payments - .into_iter() - .map(|payment| (payment.address.into(), payment.balance.amount.0)) - .collect::>(); - - match MSignableTransaction::new( - rct_type, - // Use the plan ID as the outgoing view key - Zeroizing::new(*plan_id), - inputs_actual, - payments, - Change::fingerprintable(change.as_ref().map(|change| change.clone().into())), - vec![], - fee_rate, - ) { - Ok(signable) => Ok(Some({ - if calculating_fee { - MakeSignableTransactionResult::Fee(signable.necessary_fee()) - } else { - MakeSignableTransactionResult::SignableTransaction(signable) - } - })), - Err(e) => match e { - SendError::UnsupportedRctType => { - panic!("trying to use an RctType unsupported by monero-wallet") - } - SendError::NoInputs | - SendError::InvalidDecoyQuantity | - SendError::NoOutputs | - SendError::TooManyOutputs | - SendError::NoChange | - SendError::TooMuchArbitraryData | - SendError::TooLargeTransaction | - SendError::WrongPrivateKey => { - panic!("created an invalid Monero transaction: {e}"); - } - SendError::MultiplePaymentIds => { - panic!("multiple payment IDs despite not supporting integrated addresses"); - } - SendError::NotEnoughFunds { inputs, outputs, necessary_fee } => { - log::debug!( - "Monero NotEnoughFunds. inputs: {:?}, outputs: {:?}, necessary_fee: {necessary_fee:?}", - inputs, - outputs - ); - match necessary_fee { - Some(necessary_fee) => { - // If we're solely calculating the fee, return the fee this TX will cost - if calculating_fee { - Ok(Some(MakeSignableTransactionResult::Fee(necessary_fee))) - } else { - // If we're actually trying to make the TX, return None - Ok(None) - } - } - // We didn't have enough funds to even cover the outputs - None => { - // Ensure we're not misinterpreting this - assert!(outputs > inputs); - Ok(None) - } - } - } - SendError::MaliciousSerialization | SendError::ClsagError(_) | SendError::FrostError(_) => { - panic!("supposedly unreachable (at this time) Monero error: {e}"); - } - }, - } - } - #[cfg(test)] fn test_view_pair() -> ViewPair { ViewPair::new(*EdwardsPoint::generator(), Zeroizing::new(Scalar::ONE.0)).unwrap() diff --git a/processor/monero/src/main.rs b/processor/monero/src/main.rs index eda24b56..5b32e0f1 100644 --- a/processor/monero/src/main.rs +++ b/processor/monero/src/main.rs @@ -15,6 +15,8 @@ mod key_gen; use crate::key_gen::KeyGenParams; mod rpc; use rpc::Rpc; + +mod decoys; /* mod scheduler; use scheduler::Scheduler; diff --git a/processor/monero/src/rpc.rs b/processor/monero/src/rpc.rs index 9244b23f..58e6cf8b 100644 --- a/processor/monero/src/rpc.rs +++ b/processor/monero/src/rpc.rs @@ -5,6 +5,7 @@ use monero_simple_request_rpc::SimpleRequestRpc; use serai_client::primitives::{NetworkId, Coin, Amount}; +use serai_db::Db; use scanner::ScannerFeed; use signers::TransactionPublisher; @@ -14,11 +15,12 @@ use crate::{ }; #[derive(Clone)] -pub(crate) struct Rpc { +pub(crate) struct Rpc { + pub(crate) db: D, pub(crate) rpc: SimpleRequestRpc, } -impl ScannerFeed for Rpc { +impl ScannerFeed for Rpc { const NETWORK: NetworkId = NetworkId::Monero; // Outputs aren't spendable until 10 blocks later due to the 10-block lock // Since we assumed scanned outputs are spendable, that sets a minimum confirmation depth of 10 @@ -37,16 +39,15 @@ impl ScannerFeed for Rpc { &self, ) -> impl Send + Future> { async move { - Ok( - self - .rpc - .get_height() - .await? - .checked_sub(1) - .expect("connected to an invalid Monero RPC") - .try_into() - .unwrap(), - ) + // The decoys task only indexes finalized blocks + crate::decoys::NextToIndexBlock::get(&self.db) + .ok_or_else(|| { + RpcError::InternalError("decoys task hasn't indexed any blocks yet".to_string()) + })? + .checked_sub(1) + .ok_or_else(|| { + RpcError::InternalError("only the genesis block has been indexed".to_string()) + }) } } @@ -127,7 +128,7 @@ impl ScannerFeed for Rpc { } } -impl TransactionPublisher for Rpc { +impl TransactionPublisher for Rpc { type EphemeralError = RpcError; fn publish( diff --git a/processor/monero/src/scheduler.rs b/processor/monero/src/scheduler.rs index 25f17c64..7666ec4f 100644 --- a/processor/monero/src/scheduler.rs +++ b/processor/monero/src/scheduler.rs @@ -1,3 +1,144 @@ +async fn make_signable_transaction( +block_number: usize, +plan_id: &[u8; 32], +inputs: &[Output], +payments: &[Payment], +change: &Option
, +calculating_fee: bool, +) -> Result, NetworkError> { +for payment in payments { + assert_eq!(payment.balance.coin, Coin::Monero); +} + +// TODO2: Use an fee representative of several blocks, cached inside Self +let block_for_fee = self.get_block(block_number).await?; +let fee_rate = self.median_fee(&block_for_fee).await?; + +// Determine the RCT proofs to make based off the hard fork +// TODO: Make a fn for this block which is duplicated with tests +let rct_type = match block_for_fee.header.hardfork_version { + 14 => RctType::ClsagBulletproof, + 15 | 16 => RctType::ClsagBulletproofPlus, + _ => panic!("Monero hard forked and the processor wasn't updated for it"), +}; + +let mut transcript = + RecommendedTranscript::new(b"Serai Processor Monero Transaction Transcript"); +transcript.append_message(b"plan", plan_id); + +// All signers need to select the same decoys +// All signers use the same height and a seeded RNG to make sure they do so. +let mut inputs_actual = Vec::with_capacity(inputs.len()); +for input in inputs { + inputs_actual.push( + OutputWithDecoys::fingerprintable_deterministic_new( + &mut ChaCha20Rng::from_seed(transcript.rng_seed(b"decoys")), + &self.rpc, + // TODO: Have Decoys take RctType + match rct_type { + RctType::ClsagBulletproof => 11, + RctType::ClsagBulletproofPlus => 16, + _ => panic!("selecting decoys for an unsupported RctType"), + }, + block_number + 1, + input.0.clone(), + ) + .await + .map_err(map_rpc_err)?, + ); +} + +// Monero requires at least two outputs +// If we only have one output planned, add a dummy payment +let mut payments = payments.to_vec(); +let outputs = payments.len() + usize::from(u8::from(change.is_some())); +if outputs == 0 { + return Ok(None); +} else if outputs == 1 { + payments.push(Payment { + address: Address::new( + ViewPair::new(EdwardsPoint::generator().0, Zeroizing::new(Scalar::ONE.0)) + .unwrap() + .legacy_address(MoneroNetwork::Mainnet), + ) + .unwrap(), + balance: Balance { coin: Coin::Monero, amount: Amount(0) }, + data: None, + }); +} + +let payments = payments + .into_iter() + .map(|payment| (payment.address.into(), payment.balance.amount.0)) + .collect::>(); + +match MSignableTransaction::new( + rct_type, + // Use the plan ID as the outgoing view key + Zeroizing::new(*plan_id), + inputs_actual, + payments, + Change::fingerprintable(change.as_ref().map(|change| change.clone().into())), + vec![], + fee_rate, +) { + Ok(signable) => Ok(Some({ + if calculating_fee { + MakeSignableTransactionResult::Fee(signable.necessary_fee()) + } else { + MakeSignableTransactionResult::SignableTransaction(signable) + } + })), + Err(e) => match e { + SendError::UnsupportedRctType => { + panic!("trying to use an RctType unsupported by monero-wallet") + } + SendError::NoInputs | + SendError::InvalidDecoyQuantity | + SendError::NoOutputs | + SendError::TooManyOutputs | + SendError::NoChange | + SendError::TooMuchArbitraryData | + SendError::TooLargeTransaction | + SendError::WrongPrivateKey => { + panic!("created an invalid Monero transaction: {e}"); + } + SendError::MultiplePaymentIds => { + panic!("multiple payment IDs despite not supporting integrated addresses"); + } + SendError::NotEnoughFunds { inputs, outputs, necessary_fee } => { + log::debug!( + "Monero NotEnoughFunds. inputs: {:?}, outputs: {:?}, necessary_fee: {necessary_fee:?}", + inputs, + outputs + ); + match necessary_fee { + Some(necessary_fee) => { + // If we're solely calculating the fee, return the fee this TX will cost + if calculating_fee { + Ok(Some(MakeSignableTransactionResult::Fee(necessary_fee))) + } else { + // If we're actually trying to make the TX, return None + Ok(None) + } + } + // We didn't have enough funds to even cover the outputs + None => { + // Ensure we're not misinterpreting this + assert!(outputs > inputs); + Ok(None) + } + } + } + SendError::MaliciousSerialization | SendError::ClsagError(_) | SendError::FrostError(_) => { + panic!("supposedly unreachable (at this time) Monero error: {e}"); + } + }, +} +} + + +/* use ciphersuite::{Ciphersuite, Secp256k1}; use bitcoin_serai::{ @@ -186,3 +327,4 @@ impl TransactionPlanner for Planner { } pub(crate) type Scheduler = utxo_standard_scheduler::Scheduler; +*/