p2pool: export data to an external web-server

Dumps data to JSON files which can be later served by a web-server.

CMakeLists.txt

@@ -25,6 +25,7 @@ set(HEADERS
 	src/mempool.h
 	src/p2p_server.h
 	src/p2pool.h
+	src/p2pool_api.h
 	src/params.h
 	src/pool_block.h
 	src/pool_block_parser.inl
@@ -57,6 +58,7 @@ set(SOURCES
 	src/mempool.cpp
 	src/p2p_server.cpp
 	src/p2pool.cpp
+	src/p2pool_api.cpp
 	src/params.cpp
 	src/pool_block.cpp
 	src/pow_hash.cpp

src/common.h

@@ -254,8 +254,9 @@ struct MinerData
 
 struct ChainMain
 {
-	FORCEINLINE ChainMain() : height(0), timestamp(0), reward(0), id() {}
+	FORCEINLINE ChainMain() : difficulty(0), height(0), timestamp(0), reward(0), id() {}
 
+	uint64_t difficulty;
 	uint64_t height;
 	uint64_t timestamp;
 	uint64_t reward;

src/main.cpp

@@ -32,6 +32,7 @@ static void usage()
 		"--light-mode         Don't allocate RandomX dataset, saves 2GB of RAM\n"
 		"--loglevel           Verbosity of the log, integer number between 0 and 5\n"
 		"--config             Name of the p2pool config file\n"
+		"--data-api           Path to the p2pool JSON data (use it in tandem with an external web-server)\n"
 		"--help               Show this help message\n\n"
 		"Example command line:\n\n"
 		"%s --host 127.0.0.1 --rpc-port 18081 --zmq-port 18083 --wallet YOUR_WALLET_ADDRESS --stratum [::]:3333,0.0.0.0:3333 --p2p [::]:37890,0.0.0.0:37890\n\n",

src/p2pool.cpp

@@ -30,8 +30,8 @@
 #include "params.h"
 #include "console_commands.h"
 #include "crypto.h"
+#include "p2pool_api.h"
 #include <thread>
-#include <iostream>
 
 static constexpr char log_category_prefix[] = "P2Pool ";
 constexpr int BLOCK_HEADERS_REQUIRED = 720;
@@ -93,7 +93,7 @@ p2pool::p2pool(int argc, char* argv[])
 	uv_rwlock_init_checked(&m_mainchainLock);
 	uv_mutex_init_checked(&m_submitBlockDataLock);
 
-	MinerData d;
+	m_api = m_params->m_apiPath.empty() ? nullptr : new p2pool_api(m_params->m_apiPath);
 
 	m_sideChain = new SideChain(this, type);
 	m_hasher = new RandomX_Hasher(this);
@@ -107,6 +107,7 @@ p2pool::~p2pool()
 	uv_rwlock_destroy(&m_mainchainLock);
 	uv_mutex_destroy(&m_submitBlockDataLock);
 
+	delete m_api;
 	delete m_sideChain;
 	delete m_hasher;
 	delete m_blockTemplate;
@@ -174,6 +175,8 @@ void p2pool::handle_miner_data(MinerData& data)
 	{
 		WriteLock lock(m_mainchainLock);
 
+		m_mainchainByHeight[data.height].difficulty = data.difficulty.lo;
+
 		ChainMain& c = m_mainchainByHeight[data.height - 1];
 		c.height = data.height - 1;
 		c.id = data.prev_id;
@@ -229,6 +232,7 @@ void p2pool::handle_chain_main(ChainMain& data, const char* extra)
 		WriteLock lock(m_mainchainLock);
 
 		ChainMain& c = m_mainchainByHeight[data.height];
+		c.difficulty = data.difficulty ? data.difficulty : c.difficulty;
 		c.height = data.height;
 		c.timestamp = data.timestamp;
 		c.reward = data.reward;
@@ -264,6 +268,8 @@ void p2pool::handle_chain_main(ChainMain& data, const char* extra)
 	if (!sidechain_id.empty() && side_chain().has_block(sidechain_id)) {
 		LOGINFO(0, log::LightGreen() << "BLOCK FOUND: main chain block at height " << data.height << " was mined by this p2pool" << BLOCK_FOUND);
 	}
+
+	api_update_network_stats();
 }
 
 void p2pool::submit_block_async(uint32_t template_id, uint32_t nonce, uint32_t extra_nonce)
@@ -303,6 +309,11 @@ void p2pool::submit_block_async(const std::vector<uint8_t>& blob)
 void p2pool::on_stop(uv_async_t* async)
 {
 	p2pool* pool = reinterpret_cast<p2pool*>(async->data);
+
+	if (pool->m_api) {
+		pool->m_api->on_stop();
+	}
+
 	uv_close(reinterpret_cast<uv_handle_t*>(&pool->m_submitBlockAsync), nullptr);
 	uv_close(reinterpret_cast<uv_handle_t*>(&pool->m_blockTemplateAsync), nullptr);
 	uv_close(reinterpret_cast<uv_handle_t*>(&pool->m_stopAsync), nullptr);
@@ -471,6 +482,7 @@ void p2pool::download_block_headers(uint64_t current_height)
 				if (m_serversStarted.exchange(1) == 0) {
 					m_stratumServer = new StratumServer(this);
 					m_p2pServer = new P2PServer(this);
+					api_update_network_stats();
 				}
 			}
 			else {
@@ -661,7 +673,7 @@ void p2pool::parse_get_miner_data_rpc(const char* data, size_t size)
 	download_block_headers(minerData.height);
 }
 
-bool p2pool::parse_block_header(const char* data, size_t size, ChainMain& result)
+bool p2pool::parse_block_header(const char* data, size_t size, ChainMain& c)
 {
 	rapidjson::Document doc;
 	if (doc.Parse<rapidjson::kParseCommentsFlag | rapidjson::kParseTrailingCommasFlag>(data, size).HasParseError() || !doc.IsObject()) {
@@ -681,18 +693,19 @@ bool p2pool::parse_block_header(const char* data, size_t size, ChainMain& result
 		return false;
 	}
 
-	if (!PARSE(it2->value, result, height) || !PARSE(it2->value, result, timestamp) || !parseValue(it2->value, "hash", result.id)) {
+	const auto& v = it2->value;
+	if (!PARSE(v, c, difficulty) || !PARSE(v, c, height) || !PARSE(v, c, timestamp) || !PARSE(v, c, reward) || !parseValue(v, "hash", c.id)) {
 		LOGERR(1, "parse_block_header: invalid JSON response from daemon: failed to parse 'block_header'");
 		return false;
 	}
 
 	{
 		WriteLock lock(m_mainchainLock);
-		m_mainchainByHeight[result.height] = result;
-		m_mainchainByHash[result.id] = result;
+		m_mainchainByHeight[c.height] = c;
+		m_mainchainByHash[c.id] = c;
 	}
 
-	LOGINFO(4, "parsed block header for height " << result.height);
+	LOGINFO(4, "parsed block header for height " << c.height);
 	return true;
 }
 
@@ -729,7 +742,7 @@ uint32_t p2pool::parse_block_headers_range(const char* data, size_t size)
 		}
 
 		ChainMain c;
-		if (PARSE(*i, c, height) && PARSE(*i, c, timestamp) && parseValue(*i, "hash", c.id)) {
+		if (PARSE(*i, c, difficulty) && PARSE(*i, c, height) && PARSE(*i, c, timestamp) && PARSE(*i, c, reward) && parseValue(*i, "hash", c.id)) {
 			min_height = std::min(min_height, c.height);
 			max_height = std::max(max_height, c.height);
 			m_mainchainByHeight[c.height] = c;
@@ -742,6 +755,29 @@ uint32_t p2pool::parse_block_headers_range(const char* data, size_t size)
 	return num_headers_parsed;
 }
 
+void p2pool::api_update_network_stats()
+{
+	if (!m_api) {
+		return;
+	}
+
+	ChainMain mainnet_tip;
+	{
+		ReadLock lock(m_mainchainLock);
+		mainnet_tip = m_mainchainByHash[m_minerData.prev_id];
+	}
+
+	m_api->set(p2pool_api::Category::NETWORK, "stats",
+		[this, mainnet_tip](log::Stream& s)
+		{
+			s << "{\"difficulty\":" << mainnet_tip.difficulty
+				<< ",\"hash\":\"" << mainnet_tip.id
+				<< "\",\"height\":" << mainnet_tip.height
+				<< ",\"reward\":" << mainnet_tip.reward
+				<< ",\"timestamp\":" << mainnet_tip.timestamp << "}";
+		});
+}
+
 static void on_signal(uv_signal_t* handle, int signum)
 {
 	p2pool* pool = reinterpret_cast<p2pool*>(handle->data);

src/p2pool.h

@@ -31,6 +31,7 @@ class SideChain;
 class StratumServer;
 class P2PServer;
 class ConsoleCommands;
+class p2pool_api;
 
 class p2pool : public MinerCallbackHandler
 {
@@ -85,6 +86,7 @@ private:
 
 	Params* m_params;
 
+	p2pool_api* m_api;
 	SideChain* m_sideChain;
 	RandomX_Hasher* m_hasher;
 	BlockTemplate* m_blockTemplate;
@@ -111,6 +113,8 @@ private:
 	bool parse_block_header(const char* data, size_t size, ChainMain& result);
 	uint32_t parse_block_headers_range(const char* data, size_t size);
 
+	void api_update_network_stats();
+
 	std::atomic<uint32_t> m_serversStarted{ 0 };
 	StratumServer* m_stratumServer = nullptr;
 	P2PServer* m_p2pServer = nullptr;

src/p2pool_api.cpp

@@ -0,0 +1,196 @@
+/*
+ * This file is part of the Monero P2Pool <https://github.com/SChernykh/p2pool>
+ * Copyright (c) 2021 SChernykh <https://github.com/SChernykh>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "common.h"
+#include "p2pool_api.h"
+
+#ifdef _MSC_VER
+#include <direct.h>
+#else
+#include <sys/stat.h>
+#endif
+
+static constexpr char log_category_prefix[] = "P2Pool API ";
+
+namespace p2pool {
+
+p2pool_api::p2pool_api(const std::string& api_path) : m_apiPath(api_path)
+{
+	if (m_apiPath.empty()) {
+		LOGERR(1, "api path is empty");
+		panic();
+	}
+
+	if ((m_apiPath.back() != '/')
+#ifdef _WIN32
+		&& (m_apiPath.back() != '\\')
+#endif
+		) {
+		m_apiPath += '/';
+	}
+
+	struct stat buf;
+	if (stat(m_apiPath.c_str(), &buf) != 0) {
+		LOGERR(1, "path " << m_apiPath << " doesn't exist");
+		panic();
+	}
+
+	int result = uv_async_init(uv_default_loop_checked(), &m_dumpToFileAsync, on_dump_to_file);
+	if (result) {
+		LOGERR(1, "uv_async_init failed, error " << uv_err_name(result));
+		panic();
+	}
+	m_dumpToFileAsync.data = this;
+
+	uv_mutex_init_checked(&m_dumpDataLock);
+
+	m_networkPath = m_apiPath + "network/";
+
+#ifdef _MSC_VER
+	result = _mkdir(m_networkPath.c_str());
+#else
+	result = mkdir(m_networkPath.c_str(), 0775);
+#endif
+
+	if (result < 0) {
+		result = errno;
+		if (result != EEXIST) {
+			LOGERR(1, "mkdir(" << m_networkPath << ") failed, error " << result);
+			panic();
+		}
+	}
+}
+
+p2pool_api::~p2pool_api()
+{
+	uv_mutex_destroy(&m_dumpDataLock);
+}
+
+void p2pool_api::on_stop()
+{
+	uv_close(reinterpret_cast<uv_handle_t*>(&m_dumpToFileAsync), nullptr);
+}
+
+void p2pool_api::dump_to_file_async_internal(const Category& category, const char* filename, DumpFileCallbackBase&& callback)
+{
+	std::vector<char> buf(log::Stream::BUF_SIZE + 1);
+	log::Stream s(buf.data());
+	callback(s);
+	buf.resize(s.m_pos);
+
+	std::string path;
+
+	switch (category) {
+	case Category::NETWORK: path = m_networkPath + filename;
+	}
+
+	{
+		MutexLock lock(m_dumpDataLock);
+		m_dumpData[path] = std::move(buf);
+	}
+
+	uv_async_send(&m_dumpToFileAsync);
+}
+
+void p2pool_api::dump_to_file()
+{
+	std::unordered_map<std::string, std::vector<char>> data;
+	{
+		MutexLock lock(m_dumpDataLock);
+		data = std::move(m_dumpData);
+		m_dumpData.clear();
+	}
+
+	for (auto& it : data) {
+		DumpFileWork* work = new DumpFileWork{ {}, {}, {}, it.first, std::move(it.second) };
+		work->open_req.data = work;
+		work->write_req.data = work;
+		work->close_req.data = work;
+
+		const int flags = O_WRONLY | O_CREAT | O_TRUNC
+#ifdef O_BINARY
+			| O_BINARY
+#endif
+			;
+
+		const int result = uv_fs_open(uv_default_loop_checked(), &work->open_req, it.first.c_str(), flags, 0644, on_fs_open);
+		if (result < 0) {
+			LOGWARN(4, "failed to open " << it.first << ", error " << uv_err_name(result));
+			delete work;
+		}
+	}
+}
+
+void p2pool_api::on_fs_open(uv_fs_t* req)
+{
+	DumpFileWork* work = reinterpret_cast<DumpFileWork*>(req->data);
+	const int fd = static_cast<int>(req->result);
+
+	if (fd < 0) {
+		LOGWARN(4, "failed to open " << work->name << ", error " << uv_err_name(fd));
+		uv_fs_req_cleanup(req);
+		delete work;
+		return;
+	}
+
+	uv_buf_t buf[1];
+	buf[0].base = work->buf.data();
+	buf[0].len = static_cast<uint32_t>(work->buf.size());
+
+	const int result = uv_fs_write(uv_default_loop_checked(), &work->write_req, static_cast<uv_file>(fd), buf, 1, 0, on_fs_write);
+	if (result < 0) {
+		LOGWARN(4, "failed to write to " << work->name << ", error " << uv_err_name(result));
+		uv_fs_req_cleanup(req);
+		delete work;
+		return;
+	}
+
+	uv_fs_req_cleanup(req);
+}
+
+void p2pool_api::on_fs_write(uv_fs_t* req)
+{
+	DumpFileWork* work = reinterpret_cast<DumpFileWork*>(req->data);
+
+	if (req->result < 0) {
+		LOGWARN(4, "failed to write to " << work->name << ", error " << uv_err_name(static_cast<int>(req->result)));
+	}
+
+	const int result = uv_fs_close(uv_default_loop_checked(), &work->close_req, static_cast<uv_file>(work->open_req.result), on_fs_close);
+	if (result < 0) {
+		LOGWARN(4, "failed to close " << work->name << ", error " << uv_err_name(result));
+		uv_fs_req_cleanup(req);
+		delete work;
+		return;
+	}
+
+	uv_fs_req_cleanup(req);
+}
+
+void p2pool_api::on_fs_close(uv_fs_t* req)
+{
+	DumpFileWork* work = reinterpret_cast<DumpFileWork*>(req->data);
+
+	if (req->result < 0) {
+		LOGWARN(4, "failed to close " << work->name << ", error " << uv_err_name(static_cast<int>(req->result)));
+	}
+
+	uv_fs_req_cleanup(req);
+	delete work;
+}
+
+} // namespace p2pool

src/p2pool_api.h

@@ -0,0 +1,85 @@
+/*
+ * This file is part of the Monero P2Pool <https://github.com/SChernykh/p2pool>
+ * Copyright (c) 2021 SChernykh <https://github.com/SChernykh>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "uv_util.h"
+#include <unordered_map>
+
+namespace p2pool {
+
+class p2pool_api
+{
+public:
+	explicit p2pool_api(const std::string& api_path);
+	~p2pool_api();
+
+	enum class Category {
+		NETWORK,
+	};
+
+	void on_stop();
+
+	template<typename T>
+	void set(const Category& category, const char* filename, T&& callback) { dump_to_file_async_internal(category, filename, DumpFileCallback<T>(std::move(callback))); }
+
+private:
+	static void on_dump_to_file(uv_async_t* async) { reinterpret_cast<p2pool_api*>(async->data)->dump_to_file(); }
+
+	struct DumpFileWork {
+		uv_fs_t open_req;
+		uv_fs_t write_req;
+		uv_fs_t close_req;
+
+		std::string name;
+		std::vector<char> buf;
+	};
+
+	struct DumpFileCallbackBase
+	{
+		virtual ~DumpFileCallbackBase() {}
+		virtual void operator()(log::Stream&) = 0;
+	};
+
+	template<typename T>
+	struct DumpFileCallback : public DumpFileCallbackBase
+	{
+		explicit FORCEINLINE DumpFileCallback(T&& callback) : m_callback(std::move(callback)) {}
+		void operator()(log::Stream& s) override { m_callback(s); }
+
+	private:
+		DumpFileCallback& operator=(DumpFileCallback&&) = delete;
+
+		T m_callback;
+	};
+
+	void dump_to_file_async_internal(const Category& category, const char* filename, DumpFileCallbackBase&& callback);
+	void dump_to_file();
+	static void on_fs_open(uv_fs_t* req);
+	static void on_fs_write(uv_fs_t* req);
+	static void on_fs_close(uv_fs_t* req);
+
+	std::string m_apiPath;
+	std::string m_networkPath;
+
+	uv_mutex_t m_dumpDataLock;
+	std::unordered_map<std::string, std::vector<char>> m_dumpData;
+
+	uv_async_t m_dumpToFileAsync;
+};
+
+} // namespace p2pool

src/params.cpp

@@ -63,6 +63,10 @@ Params::Params(int argc, char* argv[])
 		if ((strcmp(argv[i], "--config") == 0) && (i + 1 < argc)) {
 			m_config = argv[++i];
 		}
+
+		if ((strcmp(argv[i], "--data-api") == 0) && (i + 1 < argc)) {
+			m_apiPath = argv[++i];
+		}
 	}
 }
 

src/params.h

@@ -36,6 +36,7 @@ struct Params
 	std::string m_p2pAddresses{ "[::]:37890,0.0.0.0:37890" };
 	std::string m_p2pPeerList;
 	std::string m_config;
+	std::string m_apiPath;
 };
 
 } // namespace p2pool