diff --git a/Monero-TX-Confirm-Delay/coin-comparison.R b/Monero-TX-Confirm-Delay/coin-comparison.R new file mode 100644 index 0000000..7e73381 --- /dev/null +++ b/Monero-TX-Confirm-Delay/coin-comparison.R @@ -0,0 +1,177 @@ + +library(ggplot2) +library(data.table) +library(Cairo) + +multi.chain <- list( + xmr = readRDS(""), + ltc = readRDS(""), + bch = readRDS(""), + doge = readRDS("") +) + +for (i in names(multi.chain)) { + print(i) + print(summary(multi.chain[[i]][, + diff(sort(unique(canon.receive_time))) ])) +} + +multi.chain.tx.eligiibility <- list() + +for (i in c("P2Pool", names(multi.chain))) { + print(i) + if (i == "P2Pool") { + block.template.update.adjustment <- multi.chain[["xmr"]][, + mean(diff(sort(unique(canon.receive_time))), na.rm = TRUE)] + print(block.template.update.adjustment) + + multi.chain.tx.eligiibility[[i]] <- multi.chain[["xmr"]][ (is_p2pool), + .(coin = "P2Pool", block.template.update = + min(canon.block_receive_time - canon.receive_time - + block.template.update.adjustment, na.rm = TRUE) / 60), by = "block_height"] + # Removes transactions with no canon.receive_time. The nodes did not see + # those transactions before they appeared in a received block + # WARNING: Produces Infs when block has no transactions + } else { + block.template.update.adjustment <- multi.chain[[i]][, + mean(diff(sort(unique(canon.receive_time))), na.rm = TRUE)] + print(block.template.update.adjustment) + + multi.chain.tx.eligiibility[[i]] <- multi.chain[[i]][, + .(coin = i, block.template.update = + min(canon.block_receive_time - canon.receive_time - + block.template.update.adjustment, na.rm = TRUE) / 60), by = "block_height"] + # Removes transactions with no canon.receive_time. The nodes did not see + # those transactions before they appeared in a received block + # WARNING: Produces Infs when block has no transactions + } + +} + +multi.chain.tx.eligiibility <- data.table::rbindlist(multi.chain.tx.eligiibility) + +multi.chain.tx.eligiibility <- multi.chain.tx.eligiibility[is.finite(block.template.update), ] +multi.chain.tx.eligiibility <- multi.chain.tx.eligiibility[block.template.update < 0, block.template.update := 0] + +multi.chain.tx.eligiibility[, coin := factor(coin, levels = c("doge", "bch", "ltc", "P2Pool", "xmr"), + labels = c("Dogecoin", "Bitcoin Cash", "Litecoin", "Monero (P2Pool only)", "Monero (all blocks)"))] + + +sum.stats <- function(x) { + r <- quantile(x, probs = c(0.05, 0.25, 0.5, 0.75, 0.95)) + names(r) <- c("ymin", "lower", "middle", "upper", "ymax") + r +} +# https://stackoverflow.com/questions/21915286/r-ggplot2-geom-boxplot-with-custom-quantiles + +png("Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-boxplot.png", width = 800, height = 200) + +ggplot(multi.chain.tx.eligiibility, aes(y = coin, x = block.template.update)) + + stat_summary(fun.data = sum.stats, geom = "boxplot", position = "dodge", width = 0.75, + fill = c("#d9bd62", "#0AC18E", "#A6A9AA", "#FF6600", "#FF6600")) + + ggtitle("Boxplots of Time Between Transactions Entering the Mempool and\nBecoming Candidates for Blockchain Confirmation, by Coin") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "point", shape = 18, size = 3, color = "blue") + + scale_x_continuous(breaks = 0:10, expand = expansion(mult = c(0, 0.05))) + + expand_limits(x = 0) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + +# "#0AC18E" +# https://bitcoincash.org/graphics/ +# "#A6A9AA" +# https://www.litecoin.net/litecoin-foundation#resources +# "#d9bd62" +# https://www.reddit.com/r/dogecoin/comments/2lpwtv/what_is_the_exact_color_name_we_use_on_doge_the/ +# "#FF6600" +# Monero orange + + +png("Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-barchart.png", width = 800, height = 200) + +ggplot(multi.chain.tx.eligiibility, aes(y = coin, x = block.template.update)) + + ggtitle("Average Time Between Transactions Entering the Mempool and\nBecoming Candidates for Blockchain Confirmation, by Coin") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "bar", col = "black", + fill = c("#d9bd62", "#0AC18E", "#A6A9AA", "#FF6600", "#FF6600"), width = 0.75) + + scale_x_continuous(breaks = 0:10, expand = expansion(mult = c(0, 0.05))) + + expand_limits(x = 0) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + + + + + +multi.chain.tx.confirmation <- list() + +for (i in names(multi.chain)) { + print(i) + multi.chain.tx.confirmation[[i]] <- multi.chain[[i]][, + .(coin = i, confirm.delay = + (canon.block_receive_time - canon.receive_time) / 60)] + print(mean(multi.chain.tx.confirmation[[i]]$confirm.delay, na.rm = TRUE)) +} + + +multi.chain.tx.confirmation <- data.table::rbindlist(multi.chain.tx.confirmation) + +multi.chain.tx.confirmation <- multi.chain.tx.confirmation[! is.na(confirm.delay), ] +multi.chain.tx.confirmation <- multi.chain.tx.confirmation[confirm.delay < 0, confirm.delay := 0] + +multi.chain.tx.confirmation[, coin := factor(coin, levels = c("doge", "bch", "ltc", "xmr"), + labels = c("Dogecoin", "Bitcoin Cash", "Litecoin", "Monero"))] + +multi.chain.tx.confirmation[, as.list(summary(confirm.delay * 60)), by = "coin"] +# Summary stats of delay of confirmation time for each coin + + +sum.stats <- function(x) { + r <- quantile(x, probs = c(0.05, 0.25, 0.5, 0.75, 0.95)) + names(r) <- c("ymin", "lower", "middle", "upper", "ymax") + r +} +# https://stackoverflow.com/questions/21915286/r-ggplot2-geom-boxplot-with-custom-quantiles + +png("Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-boxplot.png", width = 800, height = 200) + +ggplot(multi.chain.tx.confirmation, aes(y = coin, x = confirm.delay)) + + stat_summary(fun.data = sum.stats, geom = "boxplot", position = "dodge", width = 0.75, + fill = c("#d9bd62", "#0AC18E", "#A6A9AA", "#FF6600")) + + ggtitle("Boxplots of Time Between Transactions Entering the Mempool and\nBeing Confirmed on the Blockchain") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "point", shape = 18, size = 3, color = "blue") + + scale_x_continuous(breaks = 0:50, expand = expansion(mult = c(0, 0.05))) + + expand_limits(x = 0) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + + +png("Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-barchart.png", width = 800, height = 200) + +ggplot(multi.chain.tx.confirmation, + aes(y = coin, x = confirm.delay)) + + ggtitle("Average Time Between Transactions Entering the Mempool and\nBeing Confirmed on the Blockchain") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "bar", col = "black", + fill = c("#d9bd62", "#0AC18E", "#A6A9AA", "#FF6600"), width = 0.75) + + scale_x_continuous(breaks = 0:50, expand = expansion(mult = c(0, 0.05))) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + diff --git a/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-barchart.png b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-barchart.png new file mode 100644 index 0000000..9ec6589 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-barchart.png differ diff --git a/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-boxplot.png b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-boxplot.png new file mode 100644 index 0000000..ce66861 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-confirmation-boxplot.png differ diff --git a/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-barchart.png b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-barchart.png new file mode 100644 index 0000000..6fd4015 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-barchart.png differ diff --git a/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-boxplot.png b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-boxplot.png new file mode 100644 index 0000000..b872c14 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/coin-comparison-tx-eligibility-boxplot.png differ diff --git a/Monero-TX-Confirm-Delay/images/mining-pool-behavior-histogram.png b/Monero-TX-Confirm-Delay/images/mining-pool-behavior-histogram.png new file mode 100644 index 0000000..6d3f604 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/mining-pool-behavior-histogram.png differ diff --git a/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-barchart.png b/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-barchart.png new file mode 100644 index 0000000..89adcfd Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-barchart.png differ diff --git a/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-boxplot.png b/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-boxplot.png new file mode 100644 index 0000000..19f2621 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-boxplot.png differ diff --git a/Monero-TX-Confirm-Delay/images/xmr-pool-delay.png b/Monero-TX-Confirm-Delay/images/xmr-pool-delay.png new file mode 100644 index 0000000..c76a65e Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/xmr-pool-delay.png differ diff --git a/Monero-TX-Confirm-Delay/images/xmr-pool-ideal.png b/Monero-TX-Confirm-Delay/images/xmr-pool-ideal.png new file mode 100644 index 0000000..b71f2d9 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/xmr-pool-ideal.png differ diff --git a/Monero-TX-Confirm-Delay/images/xmr-pool-p2pool-mix.png b/Monero-TX-Confirm-Delay/images/xmr-pool-p2pool-mix.png new file mode 100644 index 0000000..e9c5257 Binary files /dev/null and b/Monero-TX-Confirm-Delay/images/xmr-pool-p2pool-mix.png differ diff --git a/Monero-TX-Confirm-Delay/other-coin-data-prep.R b/Monero-TX-Confirm-Delay/other-coin-data-prep.R new file mode 100644 index 0000000..15465e1 --- /dev/null +++ b/Monero-TX-Confirm-Delay/other-coin-data-prep.R @@ -0,0 +1,217 @@ + + +data.dir <- "" +# Must have trailing "/" + +data.begin.time <- as.integer(as.POSIXct("2022-12-21 18:00:00 UTC")) +data.end.time <- as.integer(as.POSIXct("2023-01-18 17:59:59 UTC")) + +datasets <- c("", "") + +coin.config <- list( + ltc = "", + bch = "", + doge = "") + + +tx.time.fn <- min +block.time.fn <- median + + +for (coin in c("ltc", "bch", "doge")) { + + blockchain.conf.file <- coin.config[[coin]] + + blockchain.config <- rbch::conrpc(blockchain.conf.file) + rpcport <- readLines(blockchain.conf.file) + rpcport <- rpcport[grepl("rpcport", rpcport) ] + if (length(rpcport) > 0) { + blockchain.config@url <- paste0("http://127.0.0.1:", gsub("[^0-9]", "", rpcport)) + } + + + tx.pool <- c() + + # Check that node is responding + while(length(tx.pool) == 0) { + tx.pool <- rbch::getrawmempool(blockchain.config)@result + Sys.sleep(1) + } + + + blocks.collection <- list() + mempool.collection <- list() + + for (i in datasets) { + coin.dir <- paste0(data.dir, i, "/", coin, "/") + coin.files <- list.files(coin.dir) + coin.files <- sort(coin.files, decreasing = TRUE) + blocks.collection[[i]] <- read.csv(paste0(coin.dir, + coin.files[grepl("block-archive.*csv", coin.files)][1]), stringsAsFactors = FALSE) + mempool.collection[[i]] <- read.csv(paste0(coin.dir, + coin.files[grepl("mempool-archive.*csv", coin.files)][1]), stringsAsFactors = FALSE) + + blocks.collection[[i]] <- blocks.collection[[i]][ + blocks.collection[[i]]$block_receive_time %between% c(data.begin.time, data.end.time), ] + + blocks.collection[[i]] <- blocks.collection[[i]][ blocks.collection[[i]]$block_height != 0, ] + + mempool.collection[[i]] <- mempool.collection[[i]][ + mempool.collection[[i]]$receive_time %between% c(data.begin.time, data.end.time), ] + + colnames(blocks.collection[[i]])[colnames(blocks.collection[[i]]) != "block_hash"] <- + paste0(colnames(blocks.collection[[i]])[colnames(blocks.collection[[i]]) != "block_hash"], ".", i) + colnames(mempool.collection[[i]])[colnames(mempool.collection[[i]]) != "id_hash"] <- + paste0(colnames(mempool.collection[[i]])[colnames(mempool.collection[[i]]) != "id_hash"], ".", i) + } + + blocks <- blocks.collection[[ datasets[1] ]] + mempool <- mempool.collection[[ datasets[1] ]] + + for (i in datasets[-1]) { + blocks <- merge(blocks, blocks.collection[[i]], all = TRUE) + mempool <- merge(mempool, mempool.collection[[i]], all = TRUE) + } + + mempool$canon.receive_time <- apply(mempool[, grepl("receive_time[.]", colnames(mempool))], 1, + function(x) {tx.time.fn(x, na.rm = TRUE)} ) + + blocks$canon.block_receive_time <- apply(blocks[, grepl("block_receive_time[.]", colnames(blocks))], 1, + function(x) {block.time.fn(x, na.rm = TRUE)} ) + + mempool$canon.fee <- apply(mempool[, grepl("fee[.]", colnames(mempool))], 1, + function(x) {unique(x[!is.na(x)])} ) + + mempool$canon.weight <- apply(mempool[, grepl("weight[.]", colnames(mempool))], 1, + function(x) {unique(x[!is.na(x)])} ) + + + check.block.heights.duplicated <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 2, + function(x) {sum(duplicated(x, incomparables = NA))}) + # Check if there are "duplicate" heights, i.e. two block hashes with the same height, + # which would suggest blockchain re-orgs + stopifnot(all(check.block.heights.duplicated == 0)) + + check.block.heights.unique <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 1, + function(x) {uniqueN(x, na.rm = TRUE)}) + # Check if there are any differences in block height between same block hashes, + # which would suggest blockchain re-orgs + stopifnot(all(check.block.heights.unique == 1)) + + blocks$block_height <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 1, + function(x) {unique(na.omit(x), incomparables = NA)}) + + block_height.unique <- na.omit(unique(unlist(blocks[, grepl("block_height[.]", colnames(blocks))]))) + + all.blocks <- min(block_height.unique[block_height.unique > 0]):max(block_height.unique) + # min():max() since some blocks are "skipped" + # Need to have positive since rarely block height is corrupted in RPC response + # to "0" + + blockchain.data <- vector("list", length(all.blocks)) + + + for (i in seq_along(all.blocks)) { + + blockhash <- rbch::getblockhash(blockchain.config, height = all.blocks[i])@result + + if (coin == "doge") { + block.data <- rbch::rpcpost(blockchain.config, "getblock", plist = list(blockhash, TRUE))@result + tx_hashes <- unlist(block.data$tx) + } else { + block.data <- rbch::getblock(blockchain.config, blockhash, verbosity = "l1")@result + tx_hashes <- unlist(block.data$tx) + } + + block_reward <- rbch::getrawtransaction(blockchain.config, tx_hashes[1], + verbose = TRUE)@result$vout[[1]]$value + + if (coin == "ltc") { + tx_hashes <- tx_hashes[(-1) * 1:2] + # Remove first transaction since it is the coinbase + # Remove second LTC transaction since it is the MWEB transaction + } else { + tx_hashes <- tx_hashes[-1] + } + + + if (length(tx_hashes) > 1) { + blockchain.data[[i]] <- data.table::data.table( + block_height = all.blocks[i], + id_hash = tx_hashes, + block_num_txes = length(tx_hashes), + block_reward = block_reward + ) + } else { + blockchain.data[[i]] <- data.table::data.table( + block_height = all.blocks[i], + id_hash = "", + block_num_txes = 0L, + block_reward = block_reward + ) + } + if (all.blocks[i] %% 1000 == 0) { + cat("Block", all.blocks[i], "processed\n") + } + + } + + + + blockchain.data <- data.table::rbindlist(blockchain.data) + + blocks.filled <- merge(data.table(block_height = all.blocks), + blocks[, c("block_height", "canon.block_receive_time")], all = TRUE) + + blocks.filled$canon.block_receive_time <- zoo::na.locf(blocks.filled$canon.block_receive_time, fromLast = TRUE) + + blockchain.data <- merge(blocks.filled, blockchain.data) + + blockchain.data <- merge(blockchain.data, mempool, by = "id_hash", all = TRUE) + + + receive_time.unique <- na.omit(sort(unique(blockchain.data$canon.receive_time))) + block_receive_time.unique <- na.omit(sort(unique(blockchain.data$canon.block_receive_time))) + + earliest.confirm <- vector("list", length(block_receive_time.unique) - 1) + + for (i in seq_along(earliest.confirm)) { + receive_time.confirm = receive_time.unique[ + (receive_time.unique + 1) %between% + c(block_receive_time.unique[i] - 1, block_receive_time.unique[i + 1]) + ] + + if (length(receive_time.confirm) > 0) { + earliest.confirm[[i]] <- data.table( + earliest.possible.confirmation.time = block_receive_time.unique[i + 1], + canon.receive_time = receive_time.confirm + ) + } else { + earliest.confirm[[i]] <- data.table( + earliest.possible.confirmation.time = integer(0), + canon.receive_time = integer(0) + ) + } + } + + earliest.confirm <- data.table::rbindlist(earliest.confirm) + + blockchain.data <- merge(blockchain.data, earliest.confirm, by = "canon.receive_time", all = TRUE) + + max.receive_time.range <- apply(blockchain.data[, + grepl("^receive_time[.]", colnames(blockchain.data)), with = FALSE], 1, + function(x) {diff(range(x))}) + + cat(paste0(coin, " max.receive_time.range\n")) + cat("Summary stats:\n") + print(summary(max.range)) + cat("Quantiles:\n") + print(quantile(max.range, probs = sort(c(0.05, 0.95, (0:10)/10)), na.rm = TRUE)) + + saveRDS(blockchain.data, paste0(coin, "-blockchain-data.rds")) + +} + + + + diff --git a/Monero-TX-Confirm-Delay/pool-comparison.R b/Monero-TX-Confirm-Delay/pool-comparison.R new file mode 100644 index 0000000..d5eb657 --- /dev/null +++ b/Monero-TX-Confirm-Delay/pool-comparison.R @@ -0,0 +1,190 @@ +library(ggplot2) +library(data.table) +library(Cairo) +library(fixest) + +xmr.blockchain <- readRDS("") + + +block.template.update.adjustment <- xmr.blockchain[, + mean(diff(sort(unique(canon.receive_time))), na.rm = TRUE)] + +p2pool.confirm.delay <- xmr.blockchain[Pool == "P2Pool", + .(block.template.update = + min(canon.block_receive_time - canon.receive_time - + block.template.update.adjustment, na.rm = TRUE) ), + by = c("block_height")]$block.template.update +# Removes transactions with no canon.receive_time. The nodes did not see +# those transactions before they appeared in a received block +# WARNING: Produces Infs when block has no transactions + +mean.p2pool.confirm.delay <- mean(p2pool.confirm.delay[is.finite(p2pool.confirm.delay)]) + + +receive_time.unique <- sort(unique(xmr.blockchain$canon.receive_time)) +receive_time.unique <- receive_time.unique[is.finite(receive_time.unique)] +block_receive_time.unique <- sort(unique(xmr.blockchain$canon.block_receive_time)) +block_receive_time.unique <- block_receive_time.unique[is.finite(block_receive_time.unique)] + +p2pool.sim.confirm <- vector("list", length(block_receive_time.unique) - 1) + +for (i in seq_along(p2pool.sim.confirm)) { + receive_time.confirm = receive_time.unique[ + (receive_time.unique + 1 + mean.p2pool.confirm.delay) %between% + c(block_receive_time.unique[i] - 1, block_receive_time.unique[i + 1]) + ] + + if (length(receive_time.confirm) > 0) { + p2pool.sim.confirm[[i]] <- data.table( + p2pool.sim.confirm.time = block_receive_time.unique[i + 1], + canon.receive_time = receive_time.confirm + ) + } else { + p2pool.sim.confirm[[i]] <- data.table( + p2pool.sim.confirm.time = integer(0), + canon.receive_time = integer(0) + ) + } +} + +p2pool.sim.confirm <- data.table::rbindlist(p2pool.sim.confirm) + +xmr.blockchain <- merge(xmr.blockchain, p2pool.sim.confirm, by = "canon.receive_time", all.x = TRUE) + +xmr.blockchain[, mean(canon.block_receive_time - p2pool.sim.confirm.time, na.rm = TRUE)] + +block.reward <- unique(xmr.blockchain[, .(block_reward, Pool, is_p2pool)]) + + +block.reward[, mean(block_reward, na.rm = TRUE) / 1e12, by = "Pool"] +block.reward[, mean(block_reward, na.rm = TRUE) / 1e12, by = "is_p2pool"] + +block.reward[, (mean(block_reward[is_p2pool], na.rm = TRUE) - + mean(block_reward[!is_p2pool], na.rm = TRUE) )/ 1e12] + + + +block.reward[, Pool := relevel(factor(Pool), "other")] + +summary(feols(I(block_reward/1e12) ~ Pool, data = block.reward), vcov = "hetero") +# Check statistical significance + +block.num.txs <- unique(xmr.blockchain[, .(block_num_txes, Pool, is_p2pool)]) + + +block.num.txs[, mean(block_num_txes, na.rm = TRUE), by = "Pool"] +block.num.txs[, mean(block_num_txes, na.rm = TRUE), by = "is_p2pool"] + +block.num.txs[, (mean(block_num_txes[is_p2pool], na.rm = TRUE) - + mean(block_num_txes[!is_p2pool], na.rm = TRUE) )] + +block.num.txs[, Pool := relevel(factor(Pool), "other")] + +summary(feols(block_num_txes ~ Pool, data = block.num.txs), vcov = "hetero") +# Check statistical significance + + + +block.template.update.adjustment <- xmr.blockchain[, + mean(diff(sort(unique(canon.receive_time))))] +print(block.template.update.adjustment) + +xmr.tx.eligiibility <- xmr.blockchain[, + .(block.template.update = + min(canon.block_receive_time - canon.receive_time - + block.template.update.adjustment, na.rm = TRUE) / 60), + by = c("block_height", "Pool")] +# Removes transactions with no canon.receive_time. The nodes did not see +# those transactions before they appeared in a received block +# WARNING: Produces Infs when block has no transactions + +xmr.tx.eligiibility <- xmr.tx.eligiibility[is.finite(block.template.update), ] + +xmr.tx.eligiibility[block.template.update < 0, block.template.update := 0] + + +sum.stats <- function(x) { + r <- quantile(x, probs = c(0.05, 0.25, 0.5, 0.75, 0.95)) + names(r) <- c("ymin", "lower", "middle", "upper", "ymax") + r +} +# https://stackoverflow.com/questions/21915286/r-ggplot2-geom-boxplot-with-custom-quantiles + +png("Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-boxplot.png", width = 800, height = 400) + +ggplot(xmr.tx.eligiibility, aes(y = Pool, x = block.template.update)) + + stat_summary(fun.data = sum.stats, geom = "boxplot", position = "dodge", width = 0.75, + fill = "#FF6600") + + ggtitle("Boxplots of Time Between Transactions Entering the Mempool and\nBecoming Candidates for Blockchain Confirmation, by Mining Pool") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "point", shape = 18, size = 3, color = "blue") + + scale_x_continuous(breaks = 0:10, expand = expansion(mult = c(0, 0.05))) + + expand_limits(x = 0) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + +png("Monero-TX-Confirm-Delay/images/pool-comparison-tx-eligibility-barchart.png", width = 800, height = 400) + +ggplot(xmr.tx.eligiibility, aes(y = Pool, x = block.template.update)) + + ggtitle("Average Time Between Transactions Entering the Mempool and\nBecoming Candidates for Blockchain Confirmation, by Mining Pool") + + xlab(" Minutes github.com/Rucknium") + + stat_summary(fun = mean, geom = "bar", col = "black", fill = "#FF6600", width = 0.75) + + scale_x_continuous(breaks = 0:10, expand = expansion(mult = c(0, 0.05))) + + expand_limits(x = 0) + + theme(axis.ticks.y = element_blank(), plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_blank()) + +dev.off() + + + + + + +prev.block.time <- unique(xmr.blockchain[, .(block_height, canon.block_receive_time)]) +setorder(prev.block.time, block_height) +prev.block.time[, canon.prev.block_receive_time := shift(canon.block_receive_time, type = "lag")] + +xmr.blockchain <- merge(xmr.blockchain, + prev.block.time[, .(block_height, canon.prev.block_receive_time)], by = "block_height") + +prev.block.summary <- xmr.blockchain[, + .(elapsed = (max(canon.receive_time, na.rm = TRUE) - unique(canon.prev.block_receive_time)) / 60), + by = c("block_height", "Pool")] +# max() will produce -Inf if there are no txs in the block + +prev.block.summary[! is.finite(elapsed), elapsed := 0] +# elapsed will be NA if the block contains no transactions. Therefore, +# the amount of time that has elapsed between last block and this block's +# block template is assumed to be zero. + + +line.frame <- seq(0, 10, by = 0.01) + +png("Monero-TX-Confirm-Delay/images/mining-pool-behavior-histogram.png", width = 800, height = 800) + +plot.xlim <- c(-1, 8.5) + +ggplot(prev.block.summary, aes(elapsed)) + + geom_line(aes(x = x, y = y), data = data.frame(x = line.frame, y = dexp(line.frame, rate = 0.5))) + + geom_histogram(aes(y = ..density..), bins = diff(plot.xlim) * 60, fill = "#FF6600FF") + + ggtitle("Density Histogram of Age of Youngest Transaction in a Mined Block Minus\nAge of Previous Mined Block, by Mining Pool") + + xlab(" Minutes github.com/Rucknium") + + ylab ("Density") + + facet_wrap(~ Pool, ncol = 3, scales = "free_y") + + coord_cartesian(xlim = plot.xlim) + + scale_x_continuous(breaks = -1:10) + + theme(plot.title = element_text(size = 20), + axis.text = element_text(size = 15), axis.title.x = element_text(size = 15), + axis.title.y = element_text(size = 15), strip.text = element_text(size = 15)) + +dev.off() + + + + diff --git a/Monero-TX-Confirm-Delay/tx-delay-diagrams.R b/Monero-TX-Confirm-Delay/tx-delay-diagrams.R new file mode 100644 index 0000000..d720598 --- /dev/null +++ b/Monero-TX-Confirm-Delay/tx-delay-diagrams.R @@ -0,0 +1,195 @@ + + +library(data.table) + +set.seed(314) + +txs <- cumsum(rexp(50)) +# Transaction arrival can be modeled as a Poisson process. The time interval +# between arrival of transactions are independent exponential random variables +txs <- txs/txs[length(txs)] + +# ideal +blk.geom.ideal <- list( + list( + col = "red", + text = "Block mined\nby red pool", + tx0 = 0, + tx1 = 0.4, + block = 0.4 + ), + list( + col = "blue", + text = "Block mined\nby blue pool", + tx0 = 0.4, + tx1 = 0.7, + block = 0.7 + ), + list( + col = "darkgreen", + text = "Block mined\nby green pool", + tx0 = 0.7, + tx1 = 0.9, + block = 0.9 + ), + list( + col = "purple", + text = "Block mined\nby purple pool", + tx0 = 0.9, + tx1 = 1.2, + block = 1.2 + ), + list( + col = "brown", + text = "Block mined\nby brown pool", + tx0 = 0.9, + tx1 = 1.2, + block = 1.5 + ) +) + +# centralized pool +blk.geom.pool <- list( + list( + col = "red", + text = "Block mined\nby red pool", + tx0 = 0, + tx1 = 0.25, + block = 0.4 + ), + list( + col = "blue", + text = "Block mined\nby blue pool", + tx0 = 0.25, + tx1 = 0.4, + block = 0.7 + ), + list( + col = "darkgreen", + text = "Block mined\nby green pool", + tx0 = 0.4, + tx1 = 0.7, + block = 0.9 + ), + list( + col = "purple", + text = "Block mined\nby purple pool", + tx0 = 0.7, + tx1 = 0.9, + block = 1.2 + ), + list( + col = "brown", + text = "Block mined\nby brown pool", + tx0 = 0.9, + tx1 = 1.2, + block = 1.5 + ) +) + +# p2pool +blk.geom.p2pool <- list( + list( + col = "red", + text = "Block mined\nby red pool", + tx0 = 0, + tx1 = 0.25, + block = 0.4 + ), + list( + col = "#FF6600FF", # Monero orange + text = "Block mined\nby p2pool", + tx0 = 0.25, + tx1 = 0.67, + block = 0.7 + ), + list( + col = "darkgreen", + text = "Block mined\nby green pool", + tx0 = 0.67, + tx1 = 0.7, + block = 0.9 + ), + list( + col = "purple", + text = "Block mined\nby purple pool", + tx0 = 0.7, + tx1 = 0.9, + block = 1.2 + ), + list( + col = "brown", + text = "Block mined\nby brown pool", + tx0 = 0.9, + tx1 = 1.2, + block = 1.5 + ) +) + +blk.geoms <- list(blk.geom.ideal, blk.geom.pool, blk.geom.p2pool) + + +filenames <- c( + "Monero-TX-Confirm-Delay/images/xmr-pool-ideal.png", + "Monero-TX-Confirm-Delay/images/xmr-pool-delay.png", + "Monero-TX-Confirm-Delay/images/xmr-pool-p2pool-mix.png") + +plot.titles <- c( + "Ideal Case", + "Pool Delay Case", + "Mixed P2Pool and Pool Delay Case" +) + +for (diagram in 1:3) { + + blk.geom <- blk.geoms[[diagram]] + + png(filenames[diagram], width = 600, height = 200) + + par(mar = c(0, 0, 0, 0)) + + plot(txs, rep(0, length(txs)), + pch = 20, cex = 0.25, + bty = "n", axes = FALSE, + frame.plot = FALSE, xaxt = "n", ann = FALSE, yaxt = "n", + ylim = c(-0.35, 0.20), xlim = c(0, 1)) + + abline(h = -0.1) + text(0.5, 0.18, labels = paste0("Monero Transactions Included in a Block: ", + plot.titles[diagram]), cex = 1.2) + text(0.5, 0.1, labels = "Time") + lines(c(0.15, 0.47), c(0.1, 0.1)) + arrows(0.53, 0.1, 0.85, 0.1) + text(0.1, -0.3, labels = "github.com/Rucknium") + + ellipse.bounds <- function(x0, x1) { + list(center = mean(c(x0, x1)), width = 0.5 * diff(c(x0, x1))) + } + + for ( i in seq_along(blk.geom)) { + + ellipse.bds <- ellipse.bounds(blk.geom[[i]]$tx0, blk.geom[[i]]$tx1) + + plotrix::draw.ellipse(ellipse.bds$center, 0, + ellipse.bds$width, c(0.025), + border = blk.geom[[i]]$col) + + txs.in.block <- txs[txs %between% c(blk.geom[[i]]$tx0, blk.geom[[i]]$tx1)] + + points(blk.geom[[i]]$block, -0.1, pch = 15, col = blk.geom[[i]]$col) + text(blk.geom[[i]]$block, -0.15, col = blk.geom[[i]]$col, cex = 1, pos = 1, + labels = paste0(blk.geom[[i]]$text, ".\n\nAverage tx\nconfirm delay: ", + round(10 * mean(blk.geom[[i]]$block - txs.in.block), 1))) + lines(c(ellipse.bds$center, blk.geom[[i]]$block), c(-0.025, -0.1), col = blk.geom[[i]]$col) + lines(c(blk.geom[[i]]$tx1, blk.geom[[i]]$block), c(0, -0.1), col = blk.geom[[i]]$col) + + } + + dev.off() + +} + + + + + diff --git a/Monero-TX-Confirm-Delay/xmr-data-prep.R b/Monero-TX-Confirm-Delay/xmr-data-prep.R new file mode 100644 index 0000000..4128018 --- /dev/null +++ b/Monero-TX-Confirm-Delay/xmr-data-prep.R @@ -0,0 +1,291 @@ + +library(data.table) + +data.dir <- "" +# Must have trailing "/" + +data.begin.time <- as.integer(as.POSIXct("2022-12-21 18:00:00 UTC")) +data.end.time <- as.integer(as.POSIXct("2023-01-18 17:59:59 UTC")) + + +p2pool <- read.csv("", stringsAsFactors = FALSE) +mining.pool.labels.dir <- "" + +datasets <- c("", "", "", "", "") + +tx.time.fn <- median +block.time.fn <- median + + +mining.pool.labels.files <- list.files(mining.pool.labels.dir, full.names = TRUE) + +mining.pool.labels <- list() + +for (i in mining.pool.labels.files) { + mining.pool.labels[[i]] <- read.csv(i, stringsAsFactors = FALSE) + + orphaned.blocks <- c( + "e2936a6f13f9d5e98fd70def38a58db4af86488016edf5a34249f1c1b70ef1c7", + "d64488e574ea237e9f6c803717d2b93271b86f6bc3d374f3922cf317e8ea4fe7" + ) + + mining.pool.labels[[i]] <- mining.pool.labels[[i]][ + ! mining.pool.labels[[i]]$Id %in% orphaned.blocks, ] + # Remove known orphaned blocks + + mining.pool.labels[[i]] <- unique(mining.pool.labels[[i]][, c("Height", "Pool")]) + stopifnot( ! any(duplicated(mining.pool.labels[[i]]$Height))) + +} + +mining.pool.labels <- do.call(rbind, mining.pool.labels) +mining.pool.labels <- unique(mining.pool.labels) +rownames(mining.pool.labels) <- NULL + +stopifnot( ! any(duplicated(mining.pool.labels$Height))) + + + +blocks.collection <- list() +mempool.collection <- list() + +for (i in datasets) { + xmr.dir <- paste0(data.dir, i, "/xmr/") + xmr.files <- list.files(xmr.dir) + xmr.files <- sort(xmr.files, decreasing = TRUE) + blocks.collection[[i]] <- read.csv(paste0(xmr.dir, + xmr.files[grepl("block-archive.*csv", xmr.files)][1]), stringsAsFactors = FALSE) + mempool.collection[[i]] <- read.csv(paste0(xmr.dir, + xmr.files[grepl("mempool-archive.*csv", xmr.files)][1]), stringsAsFactors = FALSE) + + blocks.collection[[i]] <- blocks.collection[[i]][ + blocks.collection[[i]]$block_receive_time %between% c(data.begin.time, data.end.time), ] + + blocks.collection[[i]] <- blocks.collection[[i]][ blocks.collection[[i]]$block_height != 0, ] + + mempool.collection[[i]] <- mempool.collection[[i]][ + mempool.collection[[i]]$receive_time %between% c(data.begin.time, data.end.time), ] + + colnames(blocks.collection[[i]])[colnames(blocks.collection[[i]]) != "block_hash"] <- + paste0(colnames(blocks.collection[[i]])[colnames(blocks.collection[[i]]) != "block_hash"], ".", i) + colnames(mempool.collection[[i]])[colnames(mempool.collection[[i]]) != "id_hash"] <- + paste0(colnames(mempool.collection[[i]])[colnames(mempool.collection[[i]]) != "id_hash"], ".", i) +} + + +blocks <- blocks.collection[[ datasets[1] ]] +mempool <- mempool.collection[[ datasets[1] ]] + +for (i in datasets[-1]) { + blocks <- merge(blocks, blocks.collection[[i]], all = TRUE) + mempool <- merge(mempool, mempool.collection[[i]], all = TRUE) +} + +mempool$canon.receive_time <- apply(mempool[, grepl("receive_time[.]", colnames(mempool))], 1, + function(x) {tx.time.fn(x, na.rm = TRUE)} ) + +blocks$canon.block_receive_time <- apply(blocks[, grepl("block_receive_time[.]", colnames(blocks))], 1, + function(x) {block.time.fn(x, na.rm = TRUE)} ) + +mempool$canon.fee <- apply(mempool[, grepl("fee[.]", colnames(mempool))], 1, + function(x) {unique(x[!is.na(x)])} ) +# Fee is part of the data hashed for the transaction ID, so there should +# never be more than one unique fee for a given tx ID. Source: +# Section 7.4.1 of Zero to Monero 2.0 + +mempool$canon.weight <- apply(mempool[, grepl("weight[.]", colnames(mempool))], 1, + function(x) {unique(x[!is.na(x)])} ) +# Weight is implicitly part of the data hashed for the transaction ID, so there should +# never be more than one unique weight for a given tx ID. Source: +# https://libera.monerologs.net/monero-dev/20230112#c188158 + + +check.block.heights.duplicated <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 2, + function(x) {sum(duplicated(x, incomparables = NA))}) +# Check if there are "duplicate" heights, i.e. two block hashes with the same height, +# which would suggest blockchain re-orgs +stopifnot(all(check.block.heights.duplicated == 0)) + +check.block.heights.unique <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 1, + function(x) {uniqueN(x, na.rm = TRUE)}) +# Check if there are any differences in block height between same block hashes, +# which would suggest blockchain re-orgs +stopifnot(all(check.block.heights.unique == 1)) + +blocks$block_height <- apply(blocks[, grepl("block_height[.]", colnames(blocks))], 1, + function(x) {unique(na.omit(x), incomparables = NA)}) + +block_height.unique <- na.omit(unique(unlist(blocks[, grepl("block_height[.]", colnames(blocks))]))) + +all.blocks <- min(block_height.unique[block_height.unique > 0]):max(block_height.unique) +# min():max() since some blocks are "skipped" +# Need to have positive since rarely block height is corrupted in RPC response +# to "0" + + +# Modified from TownforgeR::tf_rpc_curl function +xmr.rpc <- function( + url.rpc = "http://127.0.0.1:18081/json_rpc", + method = "", + params = list(), + userpwd = "", + num.as.string = FALSE, + nonce.as.string = FALSE, + keep.trying.rpc = FALSE, + curl = RCurl::getCurlHandle(), + ... +){ + + json.ret <- RJSONIO::toJSON( + list( + jsonrpc = "2.0", + id = "0", + method = method, + params = params + ), digits = 50 + ) + + rcp.ret <- tryCatch(RCurl::postForm(url.rpc, + .opts = list( + userpwd = userpwd, + postfields = json.ret, + httpheader = c('Content-Type' = 'application/json', Accept = 'application/json') + # https://stackoverflow.com/questions/19267261/timeout-while-reading-csv-file-from-url-in-r + ), + curl = curl + ), error = function(e) {NULL}) + + if (keep.trying.rpc && length(rcp.ret) == 0) { + while (length(rcp.ret) == 0) { + rcp.ret <- tryCatch(RCurl::postForm(url.rpc, + .opts = list( + userpwd = userpwd, + postfields = json.ret, + httpheader = c('Content-Type' = 'application/json', Accept = 'application/json') + # https://stackoverflow.com/questions/19267261/timeout-while-reading-csv-file-from-url-in-r + ), + curl = curl + ), error = function(e) {NULL}) + } + } + + if (is.null(rcp.ret)) { + stop("Cannot connect to monerod. Is monerod running?") + } + + if (num.as.string) { + rcp.ret <- gsub("(: )([-0123456789.]+)([,\n\r])", "\\1\"\\2\"\\3", rcp.ret ) + } + + if (nonce.as.string & ! num.as.string) { + rcp.ret <- gsub("(\"nonce\": )([-0123456789.]+)([,\n\r])", "\\1\"\\2\"\\3", rcp.ret ) + } + + RJSONIO::fromJSON(rcp.ret, asText = TRUE) # , simplify = FALSE +} + + +curl.handle <- RCurl::getCurlHandle() + +blockchain.data <- vector("list", length(all.blocks)) + +for (i in seq_along(all.blocks)) { + + block.data <- xmr.rpc(method = "get_block", + params = list(height = all.blocks[i]), curl = curl.handle)$result + + if (length(block.data$tx_hashes) > 0) { + blockchain.data[[i]] <- data.table::data.table( + block_height = all.blocks[i], + id_hash = block.data$tx_hashes, + block_num_txes = block.data$block_header$num_txes, + block_reward = block.data$block_header$reward + ) + } else { + blockchain.data[[i]] <- data.table::data.table( + block_height = all.blocks[i], + id_hash = "", + block_num_txes = block.data$block_header$num_txes, + block_reward = block.data$block_header$reward + ) + } + if (all.blocks[i] %% 1000 == 0) { + cat("Block", all.blocks[i], "processed\n") + } +} + +blockchain.data <- data.table::rbindlist(blockchain.data) + +blocks.filled <- merge(data.table(block_height = all.blocks), + blocks[, c("block_height", "canon.block_receive_time")], all = TRUE) + +blocks.filled$canon.block_receive_time <- zoo::na.locf(blocks.filled$canon.block_receive_time, fromLast = TRUE) + +blockchain.data <- merge(blocks.filled, blockchain.data) + +blockchain.data <- merge(blockchain.data, mempool, by = "id_hash", all = TRUE) + + +receive_time.unique <- na.omit(sort(unique(blockchain.data$canon.receive_time))) +block_receive_time.unique <- na.omit(sort(unique(blockchain.data$canon.block_receive_time))) + +earliest.confirm <- vector("list", length(block_receive_time.unique) - 1) + +for (i in seq_along(earliest.confirm)) { + receive_time.confirm = receive_time.unique[ + (receive_time.unique + 1) %between% + c(block_receive_time.unique[i] - 1, block_receive_time.unique[i + 1]) + ] + + if (length(receive_time.confirm) > 0) { + earliest.confirm[[i]] <- data.table( + earliest.possible.confirmation.time = block_receive_time.unique[i + 1], + canon.receive_time = receive_time.confirm + ) + } else { + earliest.confirm[[i]] <- data.table( + earliest.possible.confirmation.time = integer(0), + canon.receive_time = integer(0) + ) + } +} + +earliest.confirm <- data.table::rbindlist(earliest.confirm) + +blockchain.data <- merge(blockchain.data, earliest.confirm, by = "canon.receive_time", all = TRUE) + +# Monero-specific data: +blockchain.data <- merge(blockchain.data, p2pool[, c("block_height", "is_p2pool")], by = "block_height", all.x = TRUE) + +colnames(mining.pool.labels)[colnames(mining.pool.labels) == "Height"] <- "block_height" + +blockchain.data <- merge(blockchain.data, mining.pool.labels[, c("block_height", "Pool")], by = "block_height", all.x = TRUE) + +blockchain.data[is.na(Pool), Pool := "other"] + +blockchain.data[(is_p2pool), Pool := "P2Pool"] + +max.receive_time.range <- apply(blockchain.data[, + grepl("^receive_time[.]", colnames(blockchain.data)), with = FALSE], 1, + function(x) {diff(range(x))}) + +cat("xmr max.receive_time.range\n") +cat("Summary stats:\n") +print(summary(max.receive_time.range)) +cat("Quantiles:\n") +print(quantile(max.receive_time.range, probs = sort(c(0.05, 0.95, (0:10)/10)), na.rm = TRUE)) + +max.block.receive_time.range <- apply(blocks[, + grepl("^block_receive_time[.]", colnames(blocks))], 1, + function(x) {diff(range(x))}) + +cat("xmr max.block.receive_time.range\n") +cat("Summary stats:\n") +print(summary(max.block.receive_time.range)) +cat("Quantiles:\n") +print(quantile(max.block.receive_time.range, probs = sort(c(0.05, 0.95, (0:10)/10)), na.rm = TRUE)) + +saveRDS(blockchain.data, "") + + +