diff --git a/docs/administration.md b/docs/administration.md index 85a06f3..37a4275 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -45,7 +45,7 @@ to put the account into the "inactive" state. Deleting accounts is not currently supported. Every admin REST request must be a `POST` that contains a JSON object with -an `auth` field and an optional `params` field: +an `auth` field (in default settings) and an optional `params` field: ```json { @@ -53,26 +53,276 @@ an `auth` field and an optional `params` field: "params":{...} } ``` -where the `params` object is specified below. +where the `params` object is specified below. The `auth` field can be omitted +if `--disable-admin-auth` is specified in the CLI arguments for the REST +server. -## Commands +## Commands (of Admin REST API) A subset of admin commands are available via admin REST API - the remainder are initially omitted for security purposes. The commands available via REST are: - * **accept_requests**: `{"type": "import"|"create", "addresses":[...]}` - * **add_account**: `{"address": ..., "key": ...}` - * **list_accounts**: `{}` - * **list_requests**: `{}` - * **modify_account_status**: `{"status": "active"|"hidden"|"inactive", "addresses":[...]}` - * **reject_requests**: `{"type": "import"|"create", "addresses":[...]}` - * **rescan**: `{"height":..., "addresses":[...]}` + * [**accept_requests**](#accept_requests): `{"type": "import"|"create", "addresses":[...]}` + * [**add_account**](#add_account): `{"address": ..., "key": ...}` + * [**list_accounts**](#list_accounts): `{}` + * [**list_requests**](#list_requests): `{}` + * [**modify_account_status**](#modify_account_status): `{"status": "active"|"hidden"|"inactive", "addresses":[...]}` + * [**reject_requests**](#reject_requests): `{"type": "import"|"create", "addresses":[...]}` + * [**rescan**](#rescan): `{"height":..., "addresses":[...]}` + * [**webhook_add**](#webhook_add): `{"type":"tx-confirmation", "address":"...", "url":"...", ...}` with optional fields: + * **token**: A string to be returned when the webhook is triggered + * **payment_id**: 16 hex characters representing a unique identifier for a transaction + * [**webhook_delete**](#webhook_delete): `{"addresses":[...]}` + * [**webhook_delete_uuid**](#webhook_delete_uuid): `{"event_ids": [...]}` + * [**webhook_list**](#webhook_list): `{}` where the listed object must be the `params` field above. +### accept_requests +Accepts new account and rescan from block 0 requests in the incoming +queue. + +### add_account +Add account for view-key scanning. An example of the JSON: +```json +{ + "params": { + "address": "9uTcr6T9GURRt7UADQc2rhjg5oMYBDyoQ5jgx8nAvVvs757WwDkc2vHLPJhwZfCnfVdnWNvuuKzJe8eMVTKwadYzBrYRG5j", + "key": "deadbeef" + }, + "auth": "f50922f5fcd186eaa4bd7070b8072b66fea4fd736f06bd82df702e2314187d09" +} +``` + +### list_accounts +Request a listing of all active accounts in the datbase. The request +should looke like: +```bash +curl -v -H "Content-Type: application/json" -d '{}' http://127.0.0.1:8081/list_accounts +``` +when auth is disabled, and when enabled: +```bash +curl -v -H "Content-Type: application/json" -d '{"auth": "f50922f5fcd186eaa4bd7070b8072b66fea4fd736f06bd82df702e2314187d09"}' http://127.0.0.1:8081/list_accounts +``` +The response will look something like: +```json +{ + "active": [ + { + "address": "9wRAu3giCtKhSsVnkZJ7LLE6zqzrmMKpPg39S8aoC7T6F6GobeDpz8TcvVfTQT3ucW82oTYKG8v3ZMAeh8SZVXWwMdvwZew", + "scan_height": 2220875, + "access_time": 1681244149 + } + ] +} +``` + +### list_requests +This is a listing of all pending new account requests and all requests +to import from genesis block requests. When auth is disabled usage +looks like: +```bash +curl -v -H "Content-Type: application/json" -d '{}' http://127.0.0.1:8081/list_requests +``` +and with auth enabled looks like: +```bash +curl -v -H "Content-Type: application/json" -d '{"auth": "f50922f5fcd186eaa4bd7070b8072b66fea4fd736f06bd82df702e2314187d09"}' http://127.0.0.1:8081/list_requests +``` +### modify_account_status +This can change an account status to `active`, `inactive` or `hidden`. The +`active` state is the normal state - the account is being scanned and +returned by the API. The `inactive` state is still returned by the API, +but is no longer being scanned. The `hidden` is the current way to +"delete" an account - it is not scanned nor returned by the API. Accounts +cannot currently be deleted due to internal DB requirements. + +### reject_requests +This is the opposite of [`accept_requests`](#accept_requests) above. See +information from that endpoint on how to use this one. + +### rescan +This tells the scanner to rescan specific account(s) from the specified +height. + +### webhook_add +This is used to track a specific payment ID to an address or all general +payments to an address (where payment ID is zero). Using this endpint requires +a web address for callback purposes, a primary (not integrated!) address, and +finally the type ("tx-confirmation"). The event will remain in the database +until one of the delete commands ([webhook_delete_uuid](#webhook_delete_uuid) +or [webhook_delete](#webhook_delete)) is used to remove it. + +> The provided URL will use SSL/TLS if `https://` is prefixed in the URL and +will use plaintext if `http://` is prefixed in the URL. SSL/TLS connections +will use the system certificate authority (root-CAs) by default, and will +ignore all authority checks if `--webhook-ssl-verification none` is provided +on the command line when starting `monero-lws-daemon`. The webhook will fail +if there is a mismatch of `http` and `https` between the two servers, and +will also fail if `https` verification is mismatched. The rule is: (1) if +the callback server has SSL/TLS disabled, the webhook should use `http://`, +(2) if the callback server has a self-signed certificate, `https://` and +`--webhook-ssl-verification none` should be used, and (3) if the callback +server is using "Let's Encrypt" (or similar), then `https://` with no +additional command line flag should be used. + + +#### Initial Request to server +Example where admin authentication is required (`--disable-admin-auth` NOT +set on start which is the default): +```json +{ + "auth": "f50922f5fcd186eaa4bd7070b8072b66fea4fd736f06bd82df702e2314187d09", + "params": { + "type": "tx-confirmation", + "url": "http://127.0.0.1:7000", + "payment_id": "df034c176eca3296", + "token": "1234", + "address": "9uTcr6T9GURRt7UADQc2rhjg5oMYBDyoQ5jgx8nAvVvs757WwDkc2vHLPJhwZfCnfVdnWNvuuKzJe8eMVTKwadYzBrYRG5j" + } +} +``` + +Example where admin authentication is not required (`--disable-admin-auth` set on start): +```json +{ + "params": { + "type": "tx-confirmation", + "url": "http://127.0.0.1:7000", + "payment_id": "df034c176eca3296", + "token": "1234", + "address": "9uTcr6T9GURRt7UADQc2rhjg5oMYBDyoQ5jgx8nAvVvs757WwDkc2vHLPJhwZfCnfVdnWNvuuKzJe8eMVTKwadYzBrYRG5j" + } +} +``` + +As noted above - `payment_id` and `token` are both optional - `token` will +default to the empty string, and `payment_id` will default to zero. +##### Initial Response from Server +The server will replay all values back to the user for confirmation. An +additional field - `event_id` - is also returned which contains a globally +unique value (internally this is a 128-bit `UUID`). + +Example response: +```json +{ + "payment_id": "df034c176eca3296", + "event_id": "fa10a4db485145f1a24dc09c19a79d43", + "token": "1234", + "confirmations": 1, + "url": "http://127.0.0.1:7000" +} +``` + +If you use the `debug_database` command provided by the `monero-lws-admin` +executable, the event should be listed in the +`webhooks_by_account_id,payment_id` field of the returned JSON object. The +event will remain in the database until an explicit +[`webhook_delete_uuid`](#webhook_delete_uuid) is invoked. + +#### Callback from Server +When the event "fires" due to a transaction, the provided URL is invoked +with a JSON payload that looks like the below: + +```json +{ + "event": "tx-confirmation", + "payment_id": "df034c176eca3296", + "token": "1234", + "confirmations": 1, + "id": "fa10a4db485145f1a24dc09c19a79d43", + "tx_info": { + "id": { + "high": 0, + "low": 5550229 + }, + "block": 2192100, + "index": 0, + "amount": 4949570000, + "timestamp": 1678324181, + "tx_hash": "901f9a2a919b6312131537ff6117d56ce2c0dc1f1341b845d7667299e1ef892f", + "tx_prefix_hash": "89685cb7acb836fde30fae8be5d8b884e92706df086960d0508e146979ef80dc", + "tx_public": "54c153792e47c1da8ceb3979560c424c1928b7b4a089c1c8b3ce99c563e1d240", + "rct_mask": "f3449407dc3721299b5309c0c336a17daeebce55165ddd447ba28bbd1f46c201", + "payment_id": "df034c176eca3296", + "unlock_time": 0, + "mixin_count": 15, + "coinbase": false + } +} +``` +which is the same information provided by the user API. The database will +contain an entry in the `webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id` +field of the JSON object provided by the `debug_database` command. The +entry will be removed when the number of confirmations has been reached. + +### webhook_delete +Deletes all webhooks associated with a specific Monero primary address. + +### webhook_delete_uuid +Deletes all references to a specific webhook referenced by its UUID +(`event_id`) + +### webhook_list +This will list every webhook that is currently "listening" for +incoming transactions. If the server has auth disabled, the +request is simply: + +```bash +curl -v -H "Content-Type: application/json" -d '{}' http://127.0.0.1:8081/webhook_list +``` +and with auth enabled looks like: +```bash +curl -v -H "Content-Type: application/json" -d '{"auth": "f50922f5fcd186eaa4bd7070b8072b66fea4fd736f06bd82df702e2314187d09"}' http://127.0.0.1:8081/webhook_list +``` + +which returns a JSON object that looks like: +```json +{ + "webhooks": [ + { + "key": { + "user": 1, + "type": "tx-confirmation" + }, + "value": [ + { + "payment_id": "9bc1a59b34253896", + "event_id": "4dc201838af54dfe88686bea7e2b599f", + "token": "12345", + "confirmations": 5, + "url": "http://127.0.0.1:8082" + }, + { + "payment_id": "9bc1a59b34253896", + "event_id": "615171e477464401a1a23cdb45b3b433", + "token": "12345", + "confirmations": 5, + "url": "http://127.0.0.1:8082" + }, + { + "payment_id": "9bc1a59b34253896", + "event_id": "e64be3ad6d1647618fbd292be0485901", + "token": "this is a fresh test", + "confirmations": 1, + "url": "http://127.0.0.1:8082/foobar" + }, + { + "payment_id": "9bc1a59b34253896", + "event_id": "fe692cdf7de1453898ad453d8fabce42", + "token": "12345", + "confirmations": 5, + "url": "http://127.0.0.1:8082/foobar" + } + ] + } + ] +} +``` + # Examples ## Admin REST API +### Default Settings ```json { "auth":"6d732245002a9499b3842c0a7f9fc6b2d657c77bd612dbefa4f7f9357d08530a", @@ -84,6 +334,16 @@ where the listed object must be the `params` field above. ``` will put the listed address into the "inactive" state. +### `--disable-admin-auth` Setting +```json +{ + "params":{ + "status": "inactive", + "addresses": ["9sAejnQ9EBR1111111111111111111111111111111111AdYmVTw2Tv6L9KYkHjJ2wd737ov8ZL5QU7CJ4zV6basGP9fyno"] + } + } +``` + ## monero-lws-admin **List every active Monero address on a newline:** @@ -95,7 +355,6 @@ will put the listed address into the "inactive" state. ```bash monero-lws-admin accept_requests create $(monero-lws-admin list_requests | jq -j '.create? | .[]? | .address?+" "') ``` - # Debugging `monero-lws-admin` has a debug mode that dumps everything stored in the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bc2b0d5..6030128 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,10 +28,10 @@ include_directories(.) +add_subdirectory(wire) add_subdirectory(db) add_subdirectory(rpc) add_subdirectory(util) -add_subdirectory(wire) # For both the server and admin utility. set(monero-lws-common_sources config.cpp error.cpp) diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index 50d5300..2030ac2 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -31,4 +31,4 @@ set(monero-lws-db_headers account.h data.h fwd.h storage.h string.h) add_library(monero-lws-db ${monero-lws-db_sources} ${monero-lws-db_headers}) target_include_directories(monero-lws-db PUBLIC "${LMDB_INCLUDE}") -target_link_libraries(monero-lws-db monero::libraries ${LMDB_LIB_PATH}) +target_link_libraries(monero-lws-db monero::libraries monero-lws-common monero-lws-wire-msgpack ${LMDB_LIB_PATH}) diff --git a/src/db/data.cpp b/src/db/data.cpp index 3c6e9c9..47b82df 100644 --- a/src/db/data.cpp +++ b/src/db/data.cpp @@ -29,8 +29,11 @@ #include #include -#include "wire/crypto.h" #include "wire.h" +#include "wire/crypto.h" +#include "wire/json/write.h" +#include "wire/msgpack.h" +#include "wire/uuid.h" namespace lws { @@ -99,7 +102,7 @@ namespace db template void map_transaction_link(F& format, T& self) { - wire::object(format, WIRE_FIELD(height), WIRE_FIELD(tx_hash)); + wire::object(format, WIRE_FIELD_ID(0, height), WIRE_FIELD_ID(1, tx_hash)); } } WIRE_DEFINE_OBJECT(transaction_link, map_transaction_link); @@ -124,19 +127,19 @@ namespace db nullptr : std::addressof(payment_bytes); wire::object(dest, - wire::field("id", std::cref(self.spend_meta.id)), - wire::field("block", self.link.height), - wire::field("index", self.spend_meta.index), - wire::field("amount", self.spend_meta.amount), - wire::field("timestamp", self.timestamp), - wire::field("tx_hash", std::cref(self.link.tx_hash)), - wire::field("tx_prefix_hash", std::cref(self.tx_prefix_hash)), - wire::field("tx_public", std::cref(self.spend_meta.tx_public)), - wire::optional_field("rct_mask", rct_mask), - wire::optional_field("payment_id", payment_id), - wire::field("unlock_time", self.unlock_time), - wire::field("mixin_count", self.spend_meta.mixin_count), - wire::field("coinbase", coinbase) + wire::field<0>("id", std::cref(self.spend_meta.id)), + wire::field<1>("block", self.link.height), + wire::field<2>("index", self.spend_meta.index), + wire::field<3>("amount", self.spend_meta.amount), + wire::field<4>("timestamp", self.timestamp), + wire::field<5>("tx_hash", std::cref(self.link.tx_hash)), + wire::field<6>("tx_prefix_hash", std::cref(self.tx_prefix_hash)), + wire::field<7>("tx_public", std::cref(self.spend_meta.tx_public)), + wire::optional_field<8>("rct_mask", rct_mask), + wire::optional_field<9>("payment_id", payment_id), + wire::field<10>("unlock_time", self.unlock_time), + wire::field<11>("mixin_count", self.spend_meta.mixin_count), + wire::field<12>("coinbase", coinbase) ); } @@ -206,9 +209,99 @@ namespace db ); } + namespace + { + constexpr const char* map_webhook_type[] = {"tx-confirmation"}; + + template + void map_webhook_key(F& format, T& self) + { + wire::object(format, WIRE_FIELD_ID(0, user), WIRE_FIELD_ID(1, type)); + } + + template + void map_webhook_data(F& format, T& self) + { + wire::object(format, + WIRE_FIELD_ID(0, url), + WIRE_FIELD_ID(1, token), + WIRE_FIELD_ID(2, confirmations) + ); + } + + template + void map_webhook_value(F& format, T& self, crypto::hash8& payment_id) + { + static_assert(sizeof(payment_id) == sizeof(self.first.payment_id), "bad memcpy"); + wire::object(format, + wire::field<0>("payment_id", std::ref(payment_id)), + wire::field<1>("event_id", std::ref(self.first.event_id)), + wire::field<2>("token", std::ref(self.second.token)), + wire::field<3>("confirmations", self.second.confirmations), + wire::field<4>("url", std::ref(self.second.url)) + ); + } + } + WIRE_DEFINE_ENUM(webhook_type, map_webhook_type); + WIRE_DEFINE_OBJECT(webhook_key, map_webhook_key); + WIRE_MSGPACK_DEFINE_OBJECT(webhook_data, map_webhook_data); + + void read_bytes(wire::reader& source, webhook_value& dest) + { + crypto::hash8 payment_id{}; + map_webhook_value(source, dest, payment_id); + std::memcpy(std::addressof(dest.first.payment_id), std::addressof(payment_id), sizeof(payment_id)); + } + void write_bytes(wire::writer& dest, const webhook_value& source) + { + crypto::hash8 payment_id; + std::memcpy(std::addressof(payment_id), std::addressof(source.first.payment_id), sizeof(payment_id)); + map_webhook_value(dest, source, payment_id); + } + + void write_bytes(wire::json_writer& dest, const webhook_tx_confirmation& self) + { + crypto::hash8 payment_id; + static_assert(sizeof(payment_id) == sizeof(self.value.first.payment_id), "bad memcpy"); + std::memcpy(std::addressof(payment_id), std::addressof(self.value.first.payment_id), sizeof(payment_id)); + // to be sent to remote url + wire::object(dest, + wire::field<0>("event", std::cref(self.key.type)), + wire::field<1>("payment_id", std::cref(payment_id)), + wire::field<2>("token", std::cref(self.value.second.token)), + wire::field<3>("confirmations", std::cref(self.value.second.confirmations)), + wire::field<4>("event_id", std::cref(self.value.first.event_id)), + WIRE_FIELD_ID(5, tx_info) + ); + } + + void write_bytes(wire::json_writer& dest, const webhook_event& self) + { + crypto::hash8 payment_id; + static_assert(sizeof(payment_id) == sizeof(self.link_webhook.payment_id), "bad memcpy"); + std::memcpy(std::addressof(payment_id), std::addressof(self.link_webhook.payment_id), sizeof(payment_id)); + wire::object(dest, + wire::field<0>("tx_info", std::cref(self.link.tx)), + wire::field<1>("output_id", std::cref(self.link.out)), + wire::field<2>("payment_id", std::cref(payment_id)), + wire::field<3>("event_id", std::cref(self.link_webhook.event_id)) + ); + } + + bool operator<(const webhook_dupsort& left, const webhook_dupsort& right) noexcept + { + return left.payment_id == right.payment_id ? + std::memcmp(std::addressof(left.event_id), std::addressof(right.event_id), sizeof(left.event_id)) < 0 : + left.payment_id < right.payment_id; + } + /*! TODO consider making an `operator<` for `crypto::tx_hash`. Not known to be needed elsewhere yet. */ - + bool operator==(transaction_link const& left, transaction_link const& right) noexcept + { + return left.height == right.height && + std::memcmp(std::addressof(left.tx_hash), std::addressof(right.tx_hash), sizeof(left.tx_hash)) == 0; + } bool operator<(transaction_link const& left, transaction_link const& right) noexcept { return left.height == right.height ? diff --git a/src/db/data.h b/src/db/data.h index 59c7d3b..ebf431b 100644 --- a/src/db/data.h +++ b/src/db/data.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020, The Monero Project + // Copyright (c) 2018-2020, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -26,15 +26,19 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include #include #include #include +#include #include #include "crypto/crypto.h" #include "lmdb/util.h" #include "ringct/rctTypes.h" //! \TODO brings in lots of includes, try to remove #include "wire/fwd.h" +#include "wire/json/fwd.h" +#include "wire/msgpack/fwd.h" #include "wire/traits.h" namespace lws @@ -237,6 +241,76 @@ namespace db static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info"); void write_bytes(wire::writer& dest, const request_info& self, bool show_key = false); + enum class webhook_type : std::uint8_t + { + tx_confirmation = 0, + // unconfirmed_tx, + // new_block + // confirmed_tx, + // double_spend_tx, + // tx_confidence + }; + WIRE_DECLARE_ENUM(webhook_type); + + //! Key for upcoming webhooks or in-progress webhooks + struct webhook_key + { + account_id user; + webhook_type type; + char reserved[3]; + }; + static_assert(sizeof(webhook_key) == 4 + 1 + 3, "padding in webhook_key"); + WIRE_DECLARE_OBJECT(webhook_key); + + //! Webhook values used to sort by duplicate keys + struct webhook_dupsort + { + std::uint64_t payment_id; //!< Only used with `tx_confirmation` type. + boost::uuids::uuid event_id; + }; + static_assert(sizeof(webhook_dupsort) == 8 + 16, "padding in webhoook"); + + //! Variable length data for a webhook key/event + struct webhook_data + { + std::string url; + std::string token; + std::uint32_t confirmations; + }; + WIRE_MSGPACK_DECLARE_OBJECT(webhook_data); + + //! Compatible with lmdb::table code + using webhook_value = std::pair; + WIRE_DECLARE_OBJECT(webhook_value); + + //! Returned by DB when a webhook event "tripped" + struct webhook_tx_confirmation + { + webhook_key key; + webhook_value value; + output tx_info; + }; + void write_bytes(wire::json_writer&, const webhook_tx_confirmation&); + + //! References a specific output that triggered a webhook + struct webhook_output + { + transaction_link tx; + output_id out; + }; + + //! References all info from a webhook that triggered + struct webhook_event + { + webhook_output link; + webhook_dupsort link_webhook; + }; + void write_bytes(wire::json_writer&, const webhook_event&); + + bool operator==(transaction_link const& left, transaction_link const& right) noexcept; + bool operator<(transaction_link const& left, transaction_link const& right) noexcept; + bool operator<=(transaction_link const& left, transaction_link const& right) noexcept; + inline constexpr bool operator==(output_id left, output_id right) noexcept { return left.high == right.high && left.low == right.low; @@ -255,9 +329,28 @@ namespace db return left.high == right.high ? left.low <= right.low : left.high < right.high; } + inline constexpr bool operator<(const webhook_key& left, const webhook_key& right) noexcept + { + return left.user == right.user ? + left.type < right.type : left.user < right.user; + } + + bool operator<(const webhook_dupsort& left, const webhook_dupsort& right) noexcept; + + inline bool operator==(const webhook_output& left, const webhook_output& right) noexcept + { + return left.out == right.out && left.tx == right.tx; + } + inline bool operator<(const webhook_output& left, const webhook_output& right) noexcept + { + return left.tx == right.tx ? left.out < right.out : left.tx < right.tx; + } + inline bool operator<(const webhook_event& left, const webhook_event& right) noexcept + { + return left.link == right.link ? + left.link_webhook < right.link_webhook : left.link < right.link; + } - bool operator<(transaction_link const& left, transaction_link const& right) noexcept; - bool operator<=(transaction_link const& left, transaction_link const& right) noexcept; /*! Write `address` to `out` in base58 format using `lws::config::network` to diff --git a/src/db/storage.cpp b/src/db/storage.cpp index 5bc3ac7..041e455 100644 --- a/src/db/storage.cpp +++ b/src/db/storage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -29,7 +29,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -48,12 +50,15 @@ #include "lmdb/database.h" #include "lmdb/error.h" #include "lmdb/key_stream.h" +#include "lmdb/msgpack_table.h" #include "lmdb/table.h" #include "lmdb/util.h" #include "lmdb/value_stream.h" +#include "net/net_parse_helpers.h" // monero/contrib/epee/include #include "span.h" #include "wire/filters.h" #include "wire/json.h" +#include "wire/vector.h" namespace lws { @@ -201,6 +206,12 @@ namespace db constexpr const lmdb::basic_table requests{ "requests_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public) }; + constexpr const lmdb::msgpack_table webhooks{ + "webhooks_by_account_id,payment_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less + }; + constexpr const lmdb::basic_table events_by_account_id{ + "webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less + }; template expect check_cursor(MDB_txn& txn, MDB_dbi tbl, std::unique_ptr& cur) noexcept @@ -451,6 +462,8 @@ namespace db MDB_dbi spends; MDB_dbi images; MDB_dbi requests; + MDB_dbi webhooks; + MDB_dbi events; } tables; const unsigned create_queue_max; @@ -469,6 +482,8 @@ namespace db tables.spends = spends.open(*txn).value(); tables.images = images.open(*txn).value(); tables.requests = requests.open(*txn).value(); + tables.webhooks = webhooks.open(*txn).value(); + tables.events = events_by_account_id.open(*txn).value(); check_blockchain(*txn, tables.blocks); @@ -645,6 +660,46 @@ namespace db return requests.get_value(value); } + expect>>> + storage_reader::get_webhooks(cursor::webhooks cur) + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.webhooks, cur)); + + std::vector>> out; + + MDB_val key{}; + MDB_val value{}; + int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + for (;/* every key */;) + { + if (err) + { + if (err == MDB_NOTFOUND) + return {std::move(out)}; + return {lmdb::error(err)}; + } + + out.emplace_back(MONERO_UNWRAP(webhooks.get_key(key)), std::vector{}); + + for (; /* every dup key */ ;) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; // inner duplicate key loop + return {lmdb::error(err)}; + } + out.back().second.push_back(MONERO_UNWRAP(webhooks.get_value(value))); + err = mdb_cursor_get(cur.get(), &key, &value, MDB_NEXT_DUP); + } + err = mdb_cursor_get(cur.get(), &key, &value, MDB_NEXT); + } + + return {std::move(out)}; + } + namespace { //! `write_bytes` implementation will forward a third argument for `show_keys`. @@ -695,6 +750,14 @@ namespace db ); } + static void write_bytes(wire::json_writer& dest, const std::pair>& self) + { + wire::object(dest, + wire::field("key", std::cref(self.first)), + wire::field("value", std::cref(self.second)) + ); + } + expect storage_reader::json_debug(std::ostream& out, bool show_keys) { using boost::adaptors::reverse; @@ -713,6 +776,8 @@ namespace db cursor::spends spends_cur; cursor::images images_cur; cursor::requests requests_cur; + cursor::webhooks webhooks_cur; + cursor::webhooks events_cur; MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur)); @@ -722,6 +787,8 @@ namespace db MONERO_CHECK(check_cursor(*txn, db->tables.spends, spends_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.images, images_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.requests, requests_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.webhooks, webhooks_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.events, events_cur)); auto blocks_partial = get_blocks>(*curs.blocks_cur, 0); @@ -760,6 +827,15 @@ namespace db if (!requests_stream) return requests_stream.error(); + // This list should be smaller ... ? + const auto webhooks_data = webhooks.get_all(*webhooks_cur); + if (!webhooks_data) + return webhooks_data.error(); + + auto events_stream = events_by_account_id.get_key_stream(std::move(events_cur)); + if (!events_stream) + return events_stream.error(); + const wire::as_array_filter toggle_keys_filter{{show_keys}}; wire::json_stream_writer json_stream{out}; wire::object(json_stream, @@ -770,7 +846,9 @@ namespace db wire::field(outputs.name, wire::as_object(outputs_stream->make_range(), wire::as_integer, wire::as_array)), wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)), wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)), - wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)) + wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)), + wire::field(webhooks.name, std::cref(*webhooks_data)), + wire::field(events_by_account_id.name, wire::as_object(events_stream->make_range(), wire::as_integer, wire::as_array)) ); json_stream.finish(); @@ -955,6 +1033,42 @@ namespace db return bulk_insert(*accounts_bh_cur, new_height, epee::to_span(new_by_heights)); } + expect rollback_events(storage_internal::tables_ const& tables, MDB_txn& txn, const block_id height) + { + cursor::webhooks webhooks_cur; + cursor::events events_cur; + MONERO_CHECK(check_cursor(txn, tables.webhooks, webhooks_cur)); + MONERO_CHECK(check_cursor(txn, tables.events, events_cur)); + + MDB_val key = lmdb::to_val(height); + MDB_val value{}; + + int err = mdb_cursor_get(events_cur.get(), &key, &value, MDB_LAST); + for ( ; /* every user */ ; ) + { + for ( ; /* every event */ ;) + { + if (err) + { + if (err == MDB_NOTFOUND) + return success(); + return {lmdb::error(err)}; + } + + const webhook_event event = + MONERO_UNWRAP(events_by_account_id.get_value(value)); + + if (event.link.tx.height < height) + break; // inner for loop + + MONERO_LMDB_CHECK(mdb_cursor_del(events_cur.get(), 0)); + err = mdb_cursor_get(events_cur.get(), &key, &value, MDB_PREV); + } + err = mdb_cursor_get(events_cur.get(), &key, &value, MDB_PREV_NODUP); + } + return success(); + } + expect rollback_chain(storage_internal::tables_ const& tables, MDB_txn& txn, MDB_cursor& cur, block_id height) { MDB_val key; @@ -971,7 +1085,8 @@ namespace db if (err != MDB_NOTFOUND) return {lmdb::error(err)}; - return rollback_accounts(tables, txn, height); + MONERO_CHECK(rollback_accounts(tables, txn, height)); + return rollback_events(tables, txn, height); } template @@ -1706,22 +1821,127 @@ namespace db } return success(); } + + expect check_hooks(MDB_cursor& webhooks_cur, MDB_cursor& events_cur, const lws::account& user) + { + const account_id user_id = user.id(); + const webhook_key hook_key{user_id, webhook_type::tx_confirmation}; + + // check payment_id == x (match specific) webhooks second + for (const output& out : user.outputs()) + { + webhook_dupsort sorter{}; + static_assert(sizeof(sorter.payment_id) == sizeof(out.payment_id.short_), "bad memcpy"); + std::memcpy( + std::addressof(sorter.payment_id), std::addressof(out.payment_id.short_), sizeof(sorter.payment_id) + ); + + MDB_val key = lmdb::to_val(hook_key); + MDB_val value = lmdb::to_val(sorter); + int err = mdb_cursor_get(&webhooks_cur, &key, &value, MDB_GET_BOTH_RANGE); + + for (; /* all user/payment_id==x entries */ ;) + { + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + break; + } + const webhook_dupsort db_sorter = MONERO_UNWRAP(webhooks.get_fixed_value(value)); + if (db_sorter.payment_id != sorter.payment_id) + break; + + const webhook_event event{ + webhook_output{out.link, out.spend_meta.id}, db_sorter + }; + + MDB_val ekey = lmdb::to_val(user_id); + MDB_val evalue = lmdb::to_val(event); + MONERO_LMDB_CHECK(mdb_cursor_put(&events_cur, &ekey, &evalue, 0)); + err = mdb_cursor_get(&webhooks_cur, &key, &value, MDB_NEXT_DUP); + } + } + return success(); + } + + expect + add_ongoing_hooks(std::vector& events, MDB_cursor& webhooks_cur, MDB_cursor& outputs_cur, MDB_cursor& events_cur, const account_id user, const block_id begin, const block_id end) + { + if (begin == end) + return success(); + + const webhook_key hook_key{user, webhook_type::tx_confirmation}; + MDB_val key = lmdb::to_val(user); + MDB_val value{}; + + int err = mdb_cursor_get(&events_cur, &key, &value, MDB_SET_KEY); + for ( ; /* every ongoing event from this user */ ; ) + { + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + return success(); + } + + const webhook_event event = + MONERO_UNWRAP(events_by_account_id.get_value(value)); + + MDB_val rkey = lmdb::to_val(hook_key); + MDB_val rvalue = lmdb::to_val(event.link_webhook); + MONERO_LMDB_CHECK(mdb_cursor_get(&webhooks_cur, &rkey, &rvalue, MDB_GET_BOTH)); + + MDB_val okey = lmdb::to_val(user); + MDB_val ovalue = lmdb::to_val(event.link); + MONERO_LMDB_CHECK(mdb_cursor_get(&outputs_cur, &okey, &ovalue, MDB_GET_BOTH)); + + events.push_back( + webhook_tx_confirmation{ + MONERO_UNWRAP(webhooks.get_key(rkey)), + MONERO_UNWRAP(webhooks.get_value(rvalue)), + MONERO_UNWRAP(outputs.get_value(ovalue)) + } + ); + + const std::uint32_t requested_confirmations = + events.back().value.second.confirmations; + + events.back().value.second.confirmations = + lmdb::to_native(begin) - lmdb::to_native(event.link.tx.height) + 1; + + // copy next blocks from first + for (const auto block_num : boost::counting_range(lmdb::to_native(begin) + 1, lmdb::to_native(end))) + { + if (requested_confirmations <= events.back().value.second.confirmations) + break; + events.push_back(events.back()); + ++(events.back().value.second.confirmations); + } + if (requested_confirmations <= events.back().value.second.confirmations) + MONERO_LMDB_CHECK(mdb_cursor_del(&events_cur, 0)); + err = mdb_cursor_get(&events_cur, &key, &value, MDB_NEXT_DUP); + } + return success(); + } } // anonymous - expect storage::update(block_id height, epee::span chain, epee::span users) + expect>> storage::update(block_id height, epee::span chain, epee::span users) { if (users.empty() && chain.empty()) - return 0; - + return {std::make_pair(0, std::vector{})}; MONERO_PRECOND(!chain.empty()); MONERO_PRECOND(db != nullptr); - return db->try_write([this, height, chain, users] (MDB_txn& txn) -> expect + return db->try_write([this, height, chain, users] (MDB_txn& txn) -> expect>> { epee::span chain_copy{chain}; const std::uint64_t last_update = lmdb::to_native(height) + chain.size() - 1; + const std::uint64_t first_new = lmdb::to_native(height) + 1; + // collect all .value() errors + std::pair> updated; if (get_checkpoints().get_max_height() <= last_update) { cursor::blocks blocks_cur; @@ -1732,22 +1952,15 @@ namespace db MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_SET)); MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_LAST_DUP)); - const expect last_block = blocks.get_value(value); - if (!last_block) - return last_block.error(); - if (last_block->id < height) + const block_info last_block = MONERO_UNWRAP(blocks.get_value(value)); + if (last_block.id < height) return {lws::error::bad_blockchain}; const std::uint64_t last_same = - std::min(lmdb::to_native(last_block->id), last_update); - - const expect hash_check = - do_get_block_hash(*blocks_cur, block_id(last_same)); - if (!hash_check) - return hash_check.error(); + std::min(lmdb::to_native(last_block.id), last_update); const std::uint64_t offset = last_same - lmdb::to_native(height); - if (*hash_check != *(chain_copy.begin() + offset)) + if (MONERO_UNWRAP(do_get_block_hash(*blocks_cur, block_id(last_same))) != *(chain_copy.begin() + offset)) return {lws::error::blockchain_reorg}; chain_copy.remove_prefix(offset + 1); @@ -1764,18 +1977,21 @@ namespace db cursor::outputs outputs_cur; cursor::spends spends_cur; cursor::images images_cur; + cursor::webhooks webhooks_cur; + cursor::events events_cur; MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur)); MONERO_CHECK(check_cursor(txn, this->db->tables.outputs, outputs_cur)); MONERO_CHECK(check_cursor(txn, this->db->tables.spends, spends_cur)); MONERO_CHECK(check_cursor(txn, this->db->tables.images, images_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.webhooks, webhooks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.events, events_cur)); // for bulk inserts boost::container::static_vector heights{}; static_assert(sizeof(heights) <= 1024, "stack vector is large"); - std::size_t updated = 0; for (auto user = users.begin() ;; ++user) { if (heights.size() == heights.capacity() || user == users.end()) @@ -1812,12 +2028,8 @@ namespace db continue; // to next account } - const expect lookup = - accounts_by_address.get_value(temp_value); - if (!lookup) - return lookup.error(); - - status_key = lookup->status; + status_key = + accounts_by_address.get_value(temp_value).value().status; MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)); } expect existing = accounts.get_value(value); @@ -1840,10 +2052,154 @@ namespace db MONERO_CHECK(bulk_insert(*outputs_cur, user->id(), epee::to_span(user->outputs()))); MONERO_CHECK(add_spends(*spends_cur, *images_cur, user->id(), epee::to_span(user->spends()))); - ++updated; + MONERO_CHECK(check_hooks(*webhooks_cur, *events_cur, *user)); + MONERO_CHECK( + add_ongoing_hooks( + updated.second, *webhooks_cur, *outputs_cur, *events_cur, user->id(), block_id(first_new), block_id(last_update + 1) + ) + ); + + ++updated.first; } // ... for every account being updated ... - return updated; + return {std::move(updated)}; }); } + + expect storage::add_webhook(const webhook_type type, const account_address& address, const webhook_value& event) + { + { + epee::net_utils::http::url_content url{}; + if (event.second.url.empty() || !epee::net_utils::parse_url(event.second.url, url)) + return {error::bad_url}; + if (url.schema != "http" && url.schema != "https") + return {error::bad_url}; + } + + return db->try_write([this, type, &address, &event] (MDB_txn& txn) -> expect + { + cursor::accounts_by_address accounts_ba_cur; + cursor::webhooks webhooks_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.webhooks, webhooks_cur)); + + webhook_key key{account_id::invalid, type}; + MDB_val lmkey = lmdb::to_val(by_address_version); + MDB_val lmvalue = lmdb::to_val(address); + + { + const int err = mdb_cursor_get(accounts_ba_cur.get(), &lmkey, &lmvalue, MDB_GET_BOTH); + if (err && err != MDB_NOTFOUND) + return {lmdb::error(err)}; + if (err != MDB_NOTFOUND) + key.user = MONERO_UNWRAP(accounts_by_address.get_value(lmvalue)); + } + + if (key.user == account_id::invalid && type == webhook_type::tx_confirmation) + return {error::bad_webhook}; + + lmkey = lmdb::to_val(key); + const epee::byte_slice value = webhooks.make_value(event.first, event.second); + lmvalue = MDB_val{value.size(), const_cast(static_cast(value.data()))}; + MONERO_LMDB_CHECK(mdb_cursor_put(webhooks_cur.get(), &lmkey, &lmvalue, 0)); + return success(); + }); + } + + expect storage::clear_webhooks(const epee::span addresses) + { + if (addresses.empty()) + return success(); + + return db->try_write([this, addresses] (MDB_txn& txn) -> expect + { + cursor::accounts_by_address accounts_ba_cur; + cursor::webhooks webhooks_cur; + cursor::events events_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.webhooks, webhooks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.events, events_cur)); + + webhook_key key{account_id::invalid, webhook_type::tx_confirmation}; + for (const auto& address : addresses) + { + MDB_val lmkey = lmdb::to_val(by_address_version); + MDB_val lmvalue = lmdb::to_val(address); + + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_ba_cur.get(), &lmkey, &lmvalue, MDB_GET_BOTH)); + key.user = MONERO_UNWRAP(accounts_by_address.get_value(lmvalue)); + + lmkey = lmdb::to_val(key); + int err = mdb_cursor_get(webhooks_cur.get(), &lmkey, &lmvalue, MDB_SET); + if (!err) + MONERO_LMDB_CHECK(mdb_cursor_del(webhooks_cur.get(), MDB_NODUPDATA)); + + lmkey = lmdb::to_val(key.user); + err = mdb_cursor_get(events_cur.get(), &lmkey, &lmvalue, MDB_SET); + if (!err) + mdb_cursor_del(events_cur.get(), MDB_NODUPDATA); + } + + return success(); + }); + } + + expect storage::clear_webhooks(std::vector ids) + { + if (ids.empty()) + return success(); + + std::sort(ids.begin(), ids.end()); + + return db->try_write([this, &ids] (MDB_txn& txn) -> expect + { + cursor::webhooks webhooks_cur; + cursor::events events_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.webhooks, webhooks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.events, events_cur)); + + MDB_val key{}; + MDB_val value{}; + int err = mdb_cursor_get(webhooks_cur.get(), &key, &value, MDB_FIRST); + for ( ; /* every webhook */ ; ) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; + return {lmdb::error(err)}; + } + + const boost::uuids::uuid id = + MONERO_UNWRAP(webhooks.get_fixed_value(value)); + if (std::binary_search(ids.begin(), ids.end(), id)) + MONERO_LMDB_CHECK(mdb_cursor_del(webhooks_cur.get(), 0)); + + err = mdb_cursor_get(webhooks_cur.get(), &key, &value, MDB_NEXT); + } + + err = mdb_cursor_get(events_cur.get(), &key, &value, MDB_FIRST); + for ( ; /* every event */ ; ) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; + return {lmdb::error(err)}; + } + + const webhook_dupsort event = + MONERO_UNWRAP(events_by_account_id.get_value(value)); + if (std::binary_search(ids.begin(), ids.end(), event.event_id)) + MONERO_LMDB_CHECK(mdb_cursor_del(events_cur.get(), 0)); + + err = mdb_cursor_get(events_cur.get(), &key, &value, MDB_NEXT); + } + + return success(); + }); + } } // db } // lws diff --git a/src/db/storage.h b/src/db/storage.h index 85147f8..87a22ce 100644 --- a/src/db/storage.h +++ b/src/db/storage.h @@ -56,6 +56,9 @@ namespace db MONERO_CURSOR(blocks); MONERO_CURSOR(accounts_by_address); MONERO_CURSOR(accounts_by_height); + + MONERO_CURSOR(webhooks); + MONERO_CURSOR(events); } struct storage_internal; @@ -130,6 +133,10 @@ namespace db expect get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept; + //! \return All webhooks in the DB + expect>>> + get_webhooks(cursor::webhooks cur = nullptr); + //! Dump the contents of the database in JSON format to `out`. expect json_debug(std::ostream& out, bool show_keys); @@ -229,7 +236,28 @@ namespace db \return True iff LMDB successfully committed the update. */ - expect update(block_id height, epee::span chain, epee::span accts); + expect>> + update(block_id height, epee::span chain, epee::span accts); + + /*! + Add webhook to be tracked in the database. The webhook will "call" + the specified URL with JSON/msgpack information when the event occurs. + + \param type The webhook event type to be tracked by the DB. + \param address is required for `type == tx_confirmation`, and is not + not needed for all other types (use default construction of zeroes). + \param event Additional information for the webhook. A valid "http" + or "https" URL must be provided (or else error). All other information + is optional. + */ + expect add_webhook(webhook_type type, const account_address& address, const webhook_value& event); + + /*! Delete all webhooks associated with every value in `addresses`. This is + likely only valid for `tx_confirmation` event types. */ + expect clear_webhooks(epee::span addressses); + + //! Delete all webhooks associated with every value in `ids` + expect clear_webhooks(std::vector ids); //! `txn` must have come from a previous call on the same thread. expect start_read(lmdb::suspended_txn txn = nullptr) const; diff --git a/src/error.cpp b/src/error.cpp index 9b63d14..a1d9ec2 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -59,6 +59,10 @@ namespace lws return "Response from monerod daemon was bad/unexpected"; case error::bad_height: return "Invalid blockchain height"; + case error::bad_url: + return "Invlaid URL"; + case error::bad_webhook: + return "Invalid webhook request"; case error::blockchain_reorg: return "A blockchain reorg has been detected"; case error::configuration: diff --git a/src/error.h b/src/error.h index 0a847e7..0bb0591 100644 --- a/src/error.h +++ b/src/error.h @@ -43,6 +43,8 @@ namespace lws bad_client_tx, //!< REST client submitted invalid transaction bad_daemon_response, //!< RPC Response from daemon was invalid bad_height, //!< Invalid blockchain height + bad_url, //!< Invalid URL + bad_webhook, //!< Invalid webhook request blockchain_reorg, //!< Blockchain reorg after fetching/scanning block(s) configuration, //!< Process configuration invalid crypto_failure, //!< Cryptographic function failed diff --git a/src/lmdb/msgpack_table.h b/src/lmdb/msgpack_table.h new file mode 100644 index 0000000..72217db --- /dev/null +++ b/src/lmdb/msgpack_table.h @@ -0,0 +1,128 @@ +#pragma once + +#include + +#include "common/expect.h" // monero/src +#include "lmdb/error.h" // monero/src +#include "lmdb/table.h" // monero/src +#include "lmdb/util.h" // monero/src +#include "wire/msgpack.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options when key is fixed and value has msgpack component + template + struct msgpack_table : table + { + using key_type = K; + using fixed_value_type = V1; + using msgpack_value_type = V2; + using value_type = std::pair; + + constexpr explicit msgpack_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, flags, &lmdb::less>, value_cmp} + {} + + static expect get_key(MDB_val key) + { + if (key.mv_size != sizeof(key_type)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + key_type out; + std::memcpy(std::addressof(out), static_cast(key.mv_data), sizeof(out)); + return out; + } + + static epee::byte_slice make_value(const fixed_value_type& val1, const msgpack_value_type& val2) + { + epee::byte_stream initial; + initial.write({reinterpret_cast(std::addressof(val1)), sizeof(val1)}); + return wire_write::to_bytes(wire::msgpack_slice_writer{std::move(initial), true}, val2); + } + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template + static expect get_fixed_value(MDB_val value) noexcept + { + static_assert(std::is_same(), "bad MONERO_FIELD?"); + static_assert(std::is_pod(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size < sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast(value.mv_data) + offset, sizeof(out)); + return out; + } + + static expect get_value(MDB_val value) noexcept + { + if (value.mv_size < sizeof(fixed_value_type)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + std::pair out; + std::memcpy(std::addressof(out.first), static_cast(value.mv_data), sizeof(out.first)); + + auto msgpack_bytes = lmdb::to_byte_span(value); + msgpack_bytes.remove_prefix(sizeof(out.first)); + auto msgpack = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}); + if (!msgpack) + return msgpack.error(); + out.second = std::move(*msgpack); + + return out; + } + + //! Easier than doing another iterator .. for now :/ + static expect>>> get_all(MDB_cursor& cur) + { + MDB_val key{}; + MDB_val value{}; + int err = mdb_cursor_get(&cur, &key, &value, MDB_FIRST); + std::vector>> out; + for ( ; /* for every key */ ; ) + { + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + break; + } + + expect next_key = get_key(key); + if (!next_key) + return next_key.error(); + out.emplace_back(std::move(*next_key), std::vector{}); + + for ( ; /* for every value at key */ ; ) + { + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + break; + } + expect next_value = get_value(value); + if (!next_value) + return next_value.error(); + out.back().second.push_back(std::move(*next_value)); + err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_DUP); + } + + err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_NODUP); + } // every key + return out; + } + }; +} // lmdb diff --git a/src/rest_server.cpp b/src/rest_server.cpp index 9f75b3c..2d90994 100644 --- a/src/rest_server.cpp +++ b/src/rest_server.cpp @@ -656,7 +656,7 @@ namespace lws }; template - expect call(std::string&& root, db::storage disk, const rpc::client& gclient) + expect call(std::string&& root, db::storage disk, const rpc::client& gclient, const bool) { using request = typename E::request; using response = typename E::response; @@ -675,33 +675,35 @@ namespace lws struct admin { T params; - crypto::secret_key auth; + boost::optional auth; }; template void read_bytes(wire::json_reader& source, admin& self) { - wire::object( - source, wire::field("auth", std::ref(unwrap(unwrap(self.auth)))), WIRE_FIELD(params) - ); + wire::object(source, WIRE_OPTIONAL_FIELD(auth), WIRE_FIELD(params)); } void read_bytes(wire::json_reader& source, admin>& self) { - // params optional - wire::object(source, wire::field("auth", std::ref(unwrap(unwrap(self.auth))))); + // params optional + wire::object(source, WIRE_OPTIONAL_FIELD(auth)); } template - expect call_admin(std::string&& root, db::storage disk, const rpc::client&) + expect call_admin(std::string&& root, db::storage disk, const rpc::client&, const bool disable_auth) { using request = typename E::request; - const expect> req = wire::json::from_bytes>(std::move(root)); + expect> req = wire::json::from_bytes>(std::move(root)); if (!req) return req.error(); + if (!disable_auth) { + if (!req->auth) + return {error::account_not_found}; + db::account_address address{}; - if (!crypto::secret_key_to_public_key(req->auth, address.view_public)) + if (!crypto::secret_key_to_public_key(*(req->auth), address.view_public)) return {error::crypto_failure}; auto reader = disk.start_read(); @@ -717,14 +719,14 @@ namespace lws } wire::json_slice_writer dest{}; - MONERO_CHECK(E{}(dest, std::move(disk), req->params)); + MONERO_CHECK(E{}(dest, std::move(disk), std::move(req->params))); return dest.take_bytes(); } struct endpoint { char const* const name; - expect (*const run)(std::string&&, db::storage, rpc::client const&); + expect (*const run)(std::string&&, db::storage, rpc::client const&, bool); const unsigned max_size; }; @@ -748,7 +750,11 @@ namespace lws {"/list_requests", call_admin, 100}, {"/modify_account_status", call_admin, 50 * 1024}, {"/reject_requests", call_admin, 50 * 1024}, - {"/rescan", call_admin, 50 * 1024} + {"/rescan", call_admin, 50 * 1024}, + {"/webhook_add", call_admin, 50 * 1024}, + {"/webhook_delete", call_admin, 50 * 1024}, + {"/webhook_delete_uuid", call_admin,50 * 1024}, + {"/webhook_list", call_admin, 100} }; struct by_name_ @@ -781,13 +787,15 @@ namespace lws rpc::client client; boost::optional prefix; boost::optional admin_prefix; + bool disable_auth; - explicit internal(boost::asio::io_service& io_service, lws::db::storage disk, rpc::client client) + explicit internal(boost::asio::io_service& io_service, lws::db::storage disk, rpc::client client, const bool disable_auth) : lws::http_server_impl_base(io_service) , disk(std::move(disk)) , client(std::move(client)) , prefix() , admin_prefix() + , disable_auth(disable_auth) { assert(std::is_sorted(std::begin(endpoints), std::end(endpoints), by_name)); } @@ -853,7 +861,7 @@ namespace lws } // \TODO remove copy of json string here :/ - auto body = handler->run(std::string{query.m_body}, disk.clone(), client); + auto body = handler->run(std::string{query.m_body}, disk.clone(), client, disable_auth); if (!body) { MINFO(body.error().message() << " from " << ctx.m_remote_address.str() << " on " << handler->name); @@ -984,13 +992,13 @@ namespace lws bool any_ssl = false; for (const std::string& address : addresses) { - ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone())); + ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone()), config.disable_admin_auth); any_ssl |= init_port(ports_.back(), address, config, false); } for (const std::string& address : admin) { - ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone())); + ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone()), config.disable_admin_auth); any_ssl |= init_port(ports_.back(), address, config, true); } diff --git a/src/rest_server.h b/src/rest_server.h index 26633b5..3631c12 100644 --- a/src/rest_server.h +++ b/src/rest_server.h @@ -54,6 +54,7 @@ namespace lws std::vector access_controls; std::size_t threads; bool allow_external; + bool disable_admin_auth; }; explicit rest_server(epee::span addresses, std::vector admin, db::storage disk, rpc::client client, configuration config); diff --git a/src/rpc/admin.cpp b/src/rpc/admin.cpp index aa1635a..1563507 100644 --- a/src/rpc/admin.cpp +++ b/src/rpc/admin.cpp @@ -28,6 +28,7 @@ #include "admin.h" #include +#include #include #include #include "db/string.h" @@ -38,8 +39,17 @@ #include "wire/error.h" #include "wire/json/write.h" #include "wire/traits.h" +#include "wire/uuid.h" #include "wire/vector.h" +namespace wire +{ + static void write_bytes(wire::writer& dest, const std::pair>& self) + { + wire::object(dest, wire::field<0>("key", self.first), wire::field<1>("value", self.second)); + } +} + namespace { // Do not output "full" debug data provided by `db::data.h` header; truncate output @@ -103,11 +113,11 @@ namespace return success(); } - template - void read_addresses(wire::reader& source, T& self, U field) + template + void read_addresses(wire::reader& source, T& self, U... field) { std::vector addresses; - wire::object(source, wire::field("addresses", std::ref(addresses)), std::move(field)); + wire::object(source, wire::field("addresses", std::ref(addresses)), std::move(field)...); self.addresses.reserve(addresses.size()); for (const auto& elem : addresses) @@ -151,6 +161,31 @@ namespace lws { namespace rpc { read_addresses(source, self, WIRE_FIELD(height)); } + void read_bytes(wire::reader& source, webhook_add_req& self) + { + boost::optional address; + wire::object(source, + WIRE_FIELD_ID(0, type), + WIRE_FIELD_ID(1, url), + WIRE_OPTIONAL_FIELD_ID(2, token), + wire::optional_field<3>("address", std::ref(address)), + WIRE_OPTIONAL_FIELD_ID(4, payment_id), + WIRE_OPTIONAL_FIELD_ID(5, confirmations) + ); + if (address) + self.address = wire_unwrap(*address); + else + self.address.reset(); + } + void read_bytes(wire::reader& source, webhook_delete_req& self) + { + read_addresses(source, self); + } + + void read_bytes(wire::reader& source, webhook_delete_uuid_req& self) + { + wire::object(source, WIRE_FIELD_ID(0, event_ids)); + } expect accept_requests_::operator()(wire::writer& dest, db::storage disk, const request& req) const { @@ -195,4 +230,60 @@ namespace lws { namespace rpc { return write_addresses(dest, disk.rescan(req.height, epee::to_span(req.addresses))); } + + expect webhook_add_::operator()(wire::writer& dest, db::storage disk, request&& req) const + { + if (req.address) + { + std::uint64_t payment_id = 0; + static_assert(sizeof(payment_id) == sizeof(crypto::hash8), "invalid memcpy"); + if (req.payment_id) + std::memcpy(std::addressof(payment_id), std::addressof(*req.payment_id), sizeof(payment_id)); + db::webhook_value event{ + db::webhook_dupsort{payment_id, boost::uuids::random_generator{}()}, + db::webhook_data{ + std::move(req.url), + std::move(req.token).value_or(std::string{}), + req.confirmations.value_or(1) + } + }; + + MONERO_CHECK(disk.add_webhook(req.type, *req.address, event)); + write_bytes(dest, event); + } + else if (req.type == db::webhook_type::tx_confirmation) + return {error::bad_webhook}; + return success(); + } + + expect webhook_delete_::operator()(wire::writer& dest, db::storage disk, const request& req) const + { + MONERO_CHECK(disk.clear_webhooks(epee::to_span(req.addresses))); + wire::object(dest); // write empty object + return success(); + } + + expect webhook_del_uuid_::operator()(wire::writer& dest, db::storage disk, request req) const + { + MONERO_CHECK(disk.clear_webhooks(std::move(req.event_ids))); + wire::object(dest); // write empty object + return success(); + } + + expect webhook_list_::operator()(wire::writer& dest, db::storage disk) const + { + std::vector>> data; + { + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + auto data_ = reader->get_webhooks(); + if (!data_) + return data_.error(); + data = std::move(*data_); + } + + wire::object(dest, wire::field<0>("webhooks", data)); + return success(); + } }} // lws // rpc diff --git a/src/rpc/admin.h b/src/rpc/admin.h index 2c48745..84b0956 100644 --- a/src/rpc/admin.h +++ b/src/rpc/admin.h @@ -27,9 +27,11 @@ #pragma once +#include #include #include #include "common/expect.h" // monero/src +#include "crypto/crypto.h" // monero/src #include "db/data.h" #include "db/storage.h" #include "wire/fwd.h" @@ -68,6 +70,29 @@ namespace rpc }; void read_bytes(wire::reader&, rescan_req&); + struct webhook_add_req + { + std::string url; + boost::optional token; + boost::optional address; + boost::optional payment_id; + boost::optional confirmations; + db::webhook_type type; + }; + void read_bytes(wire::reader&, webhook_add_req&); + + struct webhook_delete_req + { + std::vector addresses; + }; + void read_bytes(wire::reader&, webhook_delete_req&); + + struct webhook_delete_uuid_req + { + std::vector event_ids; + }; + void read_bytes(wire::reader&, webhook_delete_uuid_req&); + struct accept_requests_ { @@ -122,4 +147,35 @@ namespace rpc }; constexpr const rescan_ rescan{}; + struct webhook_add_ + { + using request = webhook_add_req; + expect operator()(wire::writer& dest, db::storage disk, request&& req) const; + }; + constexpr const webhook_add_ webhook_add{}; + + struct webhook_delete_ + { + using request = webhook_delete_req; + expect operator()(wire::writer& dest, db::storage disk, const request& req) const; + }; + constexpr const webhook_delete_ webhook_delete{}; + + struct webhook_del_uuid_ + { + using request = webhook_delete_uuid_req; + expect operator()(wire::writer& dest, db::storage disk, request req) const; + }; + constexpr const webhook_del_uuid_ webhook_delete_uuid{}; + + + struct webhook_list_ + { + using request = expect; + expect operator()(wire::writer& dest, db::storage disk) const; + expect operator()(wire::writer& dest, db::storage disk, const request&) const + { return (*this)(dest, std::move(disk)); } + }; + constexpr const webhook_list_ webhook_list{}; + }} // lws // rpc diff --git a/src/scanner.cpp b/src/scanner.cpp index 8c820a0..3d81350 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020, The Monero Project + // Copyright (c) 2018-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -37,6 +37,7 @@ #include #include #include +#include #include "common/error.h" // monero/src #include "crypto/crypto.h" // monero/src @@ -47,6 +48,8 @@ #include "db/data.h" #include "error.h" #include "misc_log_ex.h" // monero/contrib/epee/include +#include "net/http_client.h" +#include "net/net_parse_helpers.h" #include "rpc/daemon_messages.h" // monero/src #include "rpc/daemon_zmq.h" #include "rpc/json.h" @@ -74,6 +77,8 @@ namespace lws namespace { + namespace net = epee::net_utils; + constexpr const std::chrono::seconds account_poll_interval{10}; constexpr const std::chrono::minutes block_rpc_timeout{2}; constexpr const std::chrono::seconds send_timeout{30}; @@ -88,13 +93,14 @@ namespace lws struct thread_data { - explicit thread_data(rpc::client client, db::storage disk, std::vector users) - : client(std::move(client)), disk(std::move(disk)), users(std::move(users)) + explicit thread_data(rpc::client client, db::storage disk, std::vector users, net::ssl_verification_t webhook_verify) + : client(std::move(client)), disk(std::move(disk)), users(std::move(users)), webhook_verify(webhook_verify) {} rpc::client client; db::storage disk; std::vector users; + net::ssl_verification_t webhook_verify; }; // until we have a signal-handler safe notification system @@ -147,6 +153,80 @@ namespace lws return true; } + void send_via_http(net::http::http_simple_client& client, boost::string_ref uri, const db::webhook_tx_confirmation& event, const net::http::fields_list& params, const std::chrono::milliseconds timeout) + { + if (uri.empty()) + uri = "/"; + + const std::string& url = event.value.second.url; + const epee::byte_slice bytes = wire::json::to_bytes(event); + const net::http::http_response_info* info = nullptr; + + MINFO("Sending webhook to " << url); + if (!client.invoke(uri, "POST", std::string{bytes.begin(), bytes.end()}, timeout, std::addressof(info), params)) + { + MERROR("Failed to invoke http request to " << url); + return; + } + + if (!info) + { + MERROR("Failed to invoke http request to " << url << ", internal error (null response ptr)"); + return; + } + + if (info->m_response_code != 200) + { + MERROR("Failed to invoke http request to " << url << ", wrong response code: " << info->m_response_code); + return; + } + } + + void send_via_http(const epee::span events, const std::chrono::milliseconds timeout, net::ssl_verification_t verify_mode) + { + if (events.empty()) + return; + + net::http::url_content url{}; + net::http::http_simple_client client{}; + + net::http::fields_list params; + params.emplace_back("Content-Type", "application/json; charset=utf-8"); + + for (const db::webhook_tx_confirmation& event : events) + { + if (event.value.second.url.empty() || !net::parse_url(event.value.second.url, url)) + { + MERROR("Bad URL for webhook event: " << event.value.second.url); + continue; + } + + const bool https = (url.schema == "https"); + if (!https && url.schema != "http") + { + MERROR("Only http or https connections: " << event.value.second.url); + continue; + } + + const net::ssl_support_t ssl_mode = https ? + net::ssl_support_t::e_ssl_support_enabled : net::ssl_support_t::e_ssl_support_disabled; + net::ssl_options_t ssl_options{ssl_mode}; + if (https) + ssl_options.verification = verify_mode; + + if (url.port == 0) + url.port = https ? 443 : 80; + + client.set_server(url.host, std::to_string(url.port), boost::none, std::move(ssl_options)); + if (client.connect(timeout)) + send_via_http(client, url.uri, event, params, timeout); + else + MERROR("Unable to send webhook to " << event.value.second.url); + + client.disconnect(); + } + } + struct by_height { bool operator()(account const& left, account const& right) const noexcept @@ -335,6 +415,7 @@ namespace lws rpc::client client{std::move(data->client)}; db::storage disk{std::move(data->disk)}; std::vector users{std::move(data->users)}; + const net::ssl_verification_t webhook_verify = data->webhook_verify; assert(!users.empty()); assert(std::is_sorted(users.begin(), users.end(), by_height{})); @@ -478,18 +559,13 @@ namespace lws blockchain.push_back(cryptonote::get_block_hash(block)); } // for each block - expect updated = disk.update( + auto updated = disk.update( users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users) ); if (!updated) { if (updated == lws::error::blockchain_reorg) { - epee::byte_stream dest{}; - { - rapidjson::Writer out{dest}; - cryptonote::json::toJsonValue(out, blocks[998]); - } MINFO("Blockchain reorg detected, resetting state"); return; } @@ -497,9 +573,10 @@ namespace lws } MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)"); - if (*updated != users.size()) + send_via_http(epee::to_span(updated->second), std::chrono::seconds{5}, webhook_verify); + if (updated->first != users.size()) { - MWARNING("Only updated " << *updated << " account(s) out of " << users.size() << ", resetting"); + MWARNING("Only updated " << updated->first << " account(s) out of " << users.size() << ", resetting"); return; } @@ -523,7 +600,7 @@ namespace lws Launches `thread_count` threads to run `scan_loop`, and then polls for active account changes in background */ - void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector users, std::vector active) + void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector users, std::vector active, const net::ssl_verification_t webhook_verify) { assert(0 < thread_count); assert(0 < users.size()); @@ -585,7 +662,7 @@ namespace lws client.watch_scan_signals(); auto data = std::make_shared( - std::move(client), disk.clone(), std::move(thread_users) + std::move(client), disk.clone(), std::move(thread_users), webhook_verify ); threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); } @@ -596,7 +673,7 @@ namespace lws client.watch_scan_signals(); auto data = std::make_shared( - std::move(client), disk.clone(), std::move(users) + std::move(client), disk.clone(), std::move(users), webhook_verify ); threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); } @@ -739,10 +816,16 @@ namespace lws return {std::move(client)}; } - void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count) + void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const boost::string_ref webhook_ssl_verification) { thread_count = std::max(std::size_t(1), thread_count); + net::ssl_verification_t webhook_verify = net::ssl_verification_t::none; + if (webhook_ssl_verification == "system_ca") + webhook_verify = net::ssl_verification_t::system_ca; + else if (webhook_ssl_verification != "none") + MONERO_THROW(lws::error::configuration, "Invalid webhook ssl verification mode"); + rpc::client client{}; for (;;) { @@ -791,7 +874,7 @@ namespace lws checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last)); } else - check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active)); + check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), webhook_verify); if (!scanner::is_running()) return; diff --git a/src/scanner.h b/src/scanner.h index c7bc7e5..14a295d 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -48,7 +48,7 @@ namespace lws static expect sync(db::storage disk, rpc::client client); //! Poll daemon until `stop()` is called, using `thread_count` threads. - static void run(db::storage disk, rpc::context ctx, std::size_t thread_count); + static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, boost::string_ref webhook_ssl_verification); //! \return True if `stop()` has never been called. static bool is_running() noexcept { return running; } diff --git a/src/server_main.cpp b/src/server_main.cpp index f6dbe26..02ba810 100644 --- a/src/server_main.cpp +++ b/src/server_main.cpp @@ -67,6 +67,8 @@ namespace const command_line::arg_descriptor create_queue_max; const command_line::arg_descriptor rates_interval; const command_line::arg_descriptor log_level; + const command_line::arg_descriptor disable_admin_auth; + const command_line::arg_descriptor webhook_ssl_verification; static std::string get_default_zmq() { @@ -99,6 +101,8 @@ namespace , create_queue_max{"create-queue-max", "Set pending create account requests maximum", 10000} , rates_interval{"exchange-rate-interval", "Retrieve exchange rates in minute intervals from cryptocompare.com if greater than 0", 0} , log_level{"log-level", "Log level [0-4]", 1} + , disable_admin_auth{"disable-admin-auth", "Make auth field optional in HTTP-REST requests", false} + , webhook_ssl_verification{"webhook-ssl-verification", "[] specify SSL verification mode for webhooks", "system_ca"} {} void prepare(boost::program_options::options_description& description) const @@ -119,6 +123,8 @@ namespace command_line::add_arg(description, create_queue_max); command_line::add_arg(description, rates_interval); command_line::add_arg(description, log_level); + command_line::add_arg(description, disable_admin_auth); + command_line::add_arg(description, webhook_ssl_verification); } }; @@ -130,6 +136,7 @@ namespace lws::rest_server::configuration rest_config; std::string daemon_rpc; std::string daemon_sub; + std::string webhook_ssl_verification; std::chrono::minutes rates_interval; std::size_t scan_threads; unsigned create_queue_max; @@ -177,10 +184,12 @@ namespace {command_line::get_arg(args, opts.rest_ssl_key), command_line::get_arg(args, opts.rest_ssl_cert)}, command_line::get_arg(args, opts.access_controls), command_line::get_arg(args, opts.rest_threads), - command_line::get_arg(args, opts.external_bind) + command_line::get_arg(args, opts.external_bind), + command_line::get_arg(args, opts.disable_admin_auth) }, command_line::get_arg(args, opts.daemon_rpc), command_line::get_arg(args, opts.daemon_sub), + command_line::get_arg(args, opts.webhook_ssl_verification), std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)}, command_line::get_arg(args, opts.scan_threads), command_line::get_arg(args, opts.create_queue_max), @@ -215,7 +224,7 @@ namespace MINFO("Listening for REST admin clients at " << address); // blocks until SIGINT - lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads); + lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, prog.webhook_ssl_verification); } } // anonymous diff --git a/src/wire/crypto.h b/src/wire/crypto.h index 1c8cf88..40ca03e 100644 --- a/src/wire/crypto.h +++ b/src/wire/crypto.h @@ -30,9 +30,19 @@ #include #include "crypto/crypto.h" // monero/src +#include "span.h" // monero/contrib/include #include "ringct/rctTypes.h" // monero/src #include "wire/traits.h" +namespace crypto +{ + template + void read_bytes(R& source, crypto::secret_key& self) + { + source.binary(epee::as_mut_byte_span(unwrap(unwrap(self)))); + } +} + namespace wire { template<> @@ -40,6 +50,11 @@ namespace wire : std::true_type {}; + template<> + struct is_blob + : std::true_type + {}; + template<> struct is_blob : std::true_type diff --git a/src/wire/field.h b/src/wire/field.h index 5e1c112..e05ae30 100644 --- a/src/wire/field.h +++ b/src/wire/field.h @@ -46,8 +46,12 @@ ::wire::field( #name , self . name ) //! The optional field has the same key name and C/C++ name -#define WIRE_OPTIONAL_FIELD(name) \ - ::wire::optional_field( #name , std::ref( self . name )) +#define WIRE_OPTIONAL_FIELD_ID(id, name) \ + ::wire::optional_field< id >( #name , std::ref( self . name )) + +//! The optional field has the same key name and C/C++ name +#define WIRE_OPTIONAL_FIELD(name) \ + WIRE_OPTIONAL_FIELD_ID(0, name) namespace wire { @@ -73,6 +77,10 @@ namespace wire static constexpr std::size_t count() noexcept { return 1; } static constexpr unsigned id() noexcept { return I; } + //! \return True if field is forced optional when `get_value().empty()`. + static constexpr bool optional_on_empty() noexcept + { return is_optional_on_empty::value; } + const char* name; T value; @@ -250,9 +258,9 @@ namespace wire template - inline constexpr bool available(const field_&) noexcept + inline constexpr bool available(const field_& elem) noexcept { - return true; + return elem.is_required() || (elem.optional_on_empty() && !wire::empty(elem.get_value())); } template inline bool available(const field_& elem) @@ -269,18 +277,5 @@ namespace wire { return elem != nullptr; } - - - // example usage : `wire::sum(std::size_t(wire::available(fields))...)` - - inline constexpr int sum() noexcept - { - return 0; - } - template - inline constexpr T sum(const T head, const U... tail) noexcept - { - return head + sum(tail...); - } } diff --git a/src/wire/msgpack/base.h b/src/wire/msgpack/base.h index 352f526..e3ac61f 100644 --- a/src/wire/msgpack/base.h +++ b/src/wire/msgpack/base.h @@ -28,6 +28,7 @@ #pragma once #include +#include #include #include #include diff --git a/src/wire/msgpack/write.h b/src/wire/msgpack/write.h index 2b755de..2467e68 100644 --- a/src/wire/msgpack/write.h +++ b/src/wire/msgpack/write.h @@ -86,8 +86,12 @@ namespace wire } protected: + msgpack_writer(epee::byte_stream&& initial, bool integer_keys, bool needs_flush) + : writer(), bytes_(std::move(initial)), expected_(1), integer_keys_(integer_keys), needs_flush_(needs_flush) + {} + msgpack_writer(bool integer_keys, bool needs_flush) - : writer(), bytes_(), expected_(1), integer_keys_(integer_keys), needs_flush_(needs_flush) + : msgpack_writer(epee::byte_stream{}, integer_keys, needs_flush) {} //! \throw std::logic_error if tree was not completed @@ -153,6 +157,10 @@ namespace wire //! Buffers entire JSON message in memory struct msgpack_slice_writer final : msgpack_writer { + msgpack_slice_writer(epee::byte_stream&& initial, bool integer_keys = false) + : msgpack_writer(std::move(initial), integer_keys, false) + {} + explicit msgpack_slice_writer(bool integer_keys = false) : msgpack_writer(integer_keys, false) {} diff --git a/src/wire/read.h b/src/wire/read.h index e57014f..901ff57 100644 --- a/src/wire/read.h +++ b/src/wire/read.h @@ -299,6 +299,24 @@ namespace wire_read unpack_variant_field(index, source, dest.get_value(), static_cast< const wire::option& >(dest)...); } + template + inline void reset_field(wire::variant_field_& dest) + {} + + template + inline void reset_field(wire::field_& dest) + { + // array fields are always optional, see `wire/field.h` + if (dest.optional_on_empty()) + wire::clear(dest.get_value()); + } + + template + inline void reset_field(wire::field_& dest) + { + dest.get_value().reset(); + } + template inline void unpack_field(std::size_t, R& source, wire::field_& dest) { @@ -377,6 +395,14 @@ namespace wire_read read_ = true; return 1 + is_required(); } + + //! Reset optional fields that were skipped + bool reset_omitted() + { + if (!is_required() && !read_) + reset_field(field_); + return true; + } }; // `expand_tracker_map` writes all `tracker` types to a table @@ -427,6 +453,7 @@ namespace wire_read throw_exception(wire::error::schema::missing_key, "", missing); } + wire::sum(fields.reset_omitted()...); source.end_object(); } } // wire_read diff --git a/src/wire/traits.h b/src/wire/traits.h index 29b0542..9f23af1 100644 --- a/src/wire/traits.h +++ b/src/wire/traits.h @@ -42,5 +42,53 @@ namespace wire template struct is_blob : std::false_type {}; -} +/*! Forces field to be optional when empty. Concept requirements for `T` when + `is_optional_on_empty::value == true`: + * must have an `empty()` method that toggles whether the associated + `wire::field_<...>` is omitted by the `wire::writer`. + * must have a `clear()` method where `empty() == true` upon completion, + used by the `wire::reader` when the `wire::field_<...>` is omitted. */ + template + struct is_optional_on_empty + : is_array // all array types in old output engine were optional when empty + {}; + + // example usage : `wire::sum(std::size_t(wire::available(fields))...)` + + inline constexpr int sum() noexcept + { + return 0; + } + template + inline constexpr T sum(const T head, const U... tail) noexcept + { + return head + sum(tail...); + } + + + //! If `T` has no `empty()` function, this function is used + template + inline constexpr bool empty(const T&...) noexcept + { + static_assert(sum(is_optional_on_empty::value...) == 0, "type needs empty method"); + return false; + } + + //! `T` has `empty()` function, use it + template + inline auto empty(const T& container) -> decltype(container.empty()) + { return container.empty(); } + + //! If `T` has no `clear()` function, this function is used + template + inline void clear(const T&...) noexcept + { + static_assert(sum(is_optional_on_empty::value...) == 0, "type needs clear method"); + } + + //! `T` has `clear()` function, use it + template + inline auto clear(T& container) -> decltype(container.clear()) + { return container.clear(); } +} diff --git a/src/wire/uuid.h b/src/wire/uuid.h new file mode 100644 index 0000000..ef59489 --- /dev/null +++ b/src/wire/uuid.h @@ -0,0 +1,39 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace wire +{ + template<> + struct is_blob + : std::true_type + {}; +} diff --git a/src/wire/write.h b/src/wire/write.h index ca1bbed..f92c210 100644 --- a/src/wire/write.h +++ b/src/wire/write.h @@ -170,15 +170,19 @@ namespace wire_write template inline bool field(W& dest, const wire::field_ elem) { - dest.key(I, elem.name); - write_bytes(dest, elem.get_value()); + // Arrays always optional, see `wire/field.h` + if (wire::available(elem)) + { + dest.key(I, elem.name); + write_bytes(dest, elem.get_value()); + } return true; } template inline bool field(W& dest, const wire::field_ elem) { - if (bool(elem.get_value())) + if (wire::available(elem)) { dest.key(I, elem.name); write_bytes(dest, *elem.get_value()); diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index b5b5997..73c6ce9 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -30,8 +30,18 @@ add_library(monero-lws-unit-framework framework.test.cpp) target_include_directories(monero-lws-unit-framework PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} "${CMAKE_SOURCE_DIR}/src") target_link_libraries(monero-lws-unit-framework) +add_subdirectory(db) +add_subdirectory(rpc) add_subdirectory(wire) add_executable(monero-lws-unit main.cpp) -target_link_libraries(monero-lws-unit monero-lws-unit-framework monero-lws-unit-wire monero-lws-unit-wire-json monero-lws-unit-wire-msgpack) +target_link_libraries( + monero-lws-unit + monero-lws-unit-db + monero-lws-unit-framework + monero-lws-unit-rpc + monero-lws-unit-wire + monero-lws-unit-wire-json + monero-lws-unit-wire-msgpack +) add_test(NAME monero-lws-unit COMMAND monero-lws-unit -v) diff --git a/tests/unit/db/CMakeLists.txt b/tests/unit/db/CMakeLists.txt new file mode 100644 index 0000000..3c956ec --- /dev/null +++ b/tests/unit/db/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright (c) 2022-2023, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +add_library(monero-lws-unit-db OBJECT storage.test.cpp webhook.test.cpp) +target_link_libraries( + monero-lws-unit-db + monero-lws-unit-framework + monero-lws-common + monero-lws-db + monero::libraries + ${Boost_PROGRAM_OPTIONS_LIBRARY} +) +#add_test(monero-lws-unit) diff --git a/tests/unit/db/storage.test.cpp b/tests/unit/db/storage.test.cpp new file mode 100644 index 0000000..28331aa --- /dev/null +++ b/tests/unit/db/storage.test.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#include "storage.test.h" + +#include +#include "common/util.h" // monero/src/ + +namespace lws { namespace db { namespace test +{ + namespace + { + boost::filesystem::path get_db_location() + { + return tools::get_default_data_dir() + "light_wallet_server_unit_testing"; + } + } + + cleanup_db::~cleanup_db() + { + boost::filesystem::remove_all(get_db_location()); + } + + storage get_fresh_db() + { + const boost::filesystem::path location = get_db_location(); + boost::filesystem::remove_all(location); + boost::filesystem::create_directories(location); + return storage::open(location.c_str(), 5); + } + + db::account make_db_account(const account_address& pubs, const crypto::secret_key& key) + { + view_key converted_key{}; + std::memcpy(std::addressof(converted_key), std::addressof(unwrap(unwrap(key))), sizeof(key)); + return { + account_id(1), account_time(0), pubs, converted_key + }; + } + + lws::account make_account(const account_address& pubs, const crypto::secret_key& key) + { + return lws::account{make_db_account(pubs, key), {}, {}}; + } +}}} // lws // db // test diff --git a/tests/unit/db/storage.test.h b/tests/unit/db/storage.test.h new file mode 100644 index 0000000..7438946 --- /dev/null +++ b/tests/unit/db/storage.test.h @@ -0,0 +1,46 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include "crypto/crypto.h" // monero/src/ +#include "db/account.h" +#include "db/data.h" +#include "db/storage.h" + +namespace lws { namespace db { namespace test +{ + struct cleanup_db + { + ~cleanup_db(); + }; + + lws::db::storage get_fresh_db(); + lws::db::account make_db_account(const lws::db::account_address& pubs, const crypto::secret_key& key); + lws::account make_account(const lws::db::account_address& pubs, const crypto::secret_key& key); +}}} // lws // db // test diff --git a/tests/unit/db/webhook.test.cpp b/tests/unit/db/webhook.test.cpp new file mode 100644 index 0000000..db2de3e --- /dev/null +++ b/tests/unit/db/webhook.test.cpp @@ -0,0 +1,210 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "framework.test.h" + +#include +#include +#include "crypto/crypto.h" // monero/src +#include "db/data.h" +#include "db/storage.h" +#include "db/storage.test.h" + +namespace +{ + bool add_out(lws::account& account, const lws::db::block_id last_id, const std::uint64_t payment_id) + { + crypto::hash8 real_id{}; + std::memcpy(std::addressof(real_id), std::addressof(payment_id), sizeof(real_id)); + return account.add_out( + lws::db::output{ + lws::db::transaction_link{ + lws::db::block_id(lmdb::to_native(last_id) + 1), + crypto::rand() + }, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 100}, + std::uint64_t(1000), + std::uint32_t(16), + std::uint32_t(1), + crypto::rand() + }, + std::uint64_t(10000000), + std::uint64_t(0), + crypto::rand(), + crypto::rand(), + crypto::rand(), + {{}, {}, {}, {}, {}, {}, {}}, + lws::db::extra_and_length(0), + lws::db::output::payment_id_{real_id} + } + ); + } +} + +LWS_CASE("db::storage::*_webhook") +{ + lws::db::account_address account{}; + crypto::secret_key view{}; + crypto::generate_keys(account.spend_public, view); + crypto::generate_keys(account.view_public, view); + + SETUP("One Account and one Webhook Database") + { + lws::db::test::cleanup_db on_scope_exit{}; + lws::db::storage db = lws::db::test::get_fresh_db(); + const lws::db::block_info last_block = + MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_last_block()); + MONERO_UNWRAP(db.add_account(account, view)); + + const boost::uuids::uuid id = boost::uuids::random_generator{}(); + { + lws::db::webhook_value value{ + lws::db::webhook_dupsort{500, id}, + lws::db::webhook_data{"http://the_url", "the_token", 3} + }; + MONERO_UNWRAP( + db.add_webhook(lws::db::webhook_type::tx_confirmation, account, std::move(value)) + ); + } + + SECTION("storage::get_webhooks()") + { + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + const auto result = MONERO_UNWRAP(reader.get_webhooks()); + EXPECT(result.size() == 1); + EXPECT(result[0].first.user == lws::db::account_id(1)); + EXPECT(result[0].first.type == lws::db::webhook_type::tx_confirmation); + EXPECT(result[0].second.size() == 1); + EXPECT(result[0].second[0].first.payment_id == 500); + EXPECT(result[0].second[0].first.event_id == id); + EXPECT(result[0].second[0].second.url == "http://the_url"); + EXPECT(result[0].second[0].second.token == "the_token"); + EXPECT(result[0].second[0].second.confirmations == 3); + } + + SECTION("storage::clear_webhooks(addresses)") + { + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).size() == 1); + MONERO_UNWRAP(db.clear_webhooks({std::addressof(account), 1})); + + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + const auto result = MONERO_UNWRAP(reader.get_webhooks()); + EXPECT(result.empty()); + } + + SECTION("storage::clear_webhooks(uuid)") + { + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).size() == 1); + MONERO_UNWRAP(db.clear_webhooks({id})); + + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + const auto result = MONERO_UNWRAP(reader.get_webhooks()); + EXPECT(result.empty()); + } + + SECTION("storage::update(...) one at a time") + { + lws::account full_account = lws::db::test::make_account(account, view); + full_account.updated(last_block.id); + EXPECT(add_out(full_account, last_block.id, 500)); + + const std::vector outs = full_account.outputs(); + EXPECT(outs.size() == 1); + + lws::db::block_info head = last_block; + for (unsigned i = 0; i < 1; ++i) + { + crypto::hash chain[2] = {head.hash, crypto::rand()}; + + auto updated = db.update(head.id, chain, {std::addressof(full_account), 1}); + EXPECT(!updated.has_error()); + EXPECT(updated->first == 1); + if (i < 3) + { + EXPECT(updated->second.size() == 1); + EXPECT(updated->second[0].key.user == lws::db::account_id(1)); + EXPECT(updated->second[0].key.type == lws::db::webhook_type::tx_confirmation); + EXPECT(updated->second[0].value.first.payment_id == 500); + EXPECT(updated->second[0].value.first.event_id == id); + EXPECT(updated->second[0].value.second.url == "http://the_url"); + EXPECT(updated->second[0].value.second.token == "the_token"); + EXPECT(updated->second[0].value.second.confirmations == i + 1); + + EXPECT(updated->second[0].tx_info.link == outs[0].link); + EXPECT(updated->second[0].tx_info.spend_meta.id == outs[0].spend_meta.id); + EXPECT(updated->second[0].tx_info.pub == outs[0].pub); + EXPECT(updated->second[0].tx_info.payment_id.short_ == outs[0].payment_id.short_); + } + else + EXPECT(updated->second.empty()); + + full_account.updated(head.id); + head = {lws::db::block_id(lmdb::to_native(head.id) + 1), chain[1]}; + } + } + + SECTION("storage::update(...) all at once") + { + const crypto::hash chain[5] = { + last_block.hash, + crypto::rand(), + crypto::rand(), + crypto::rand(), + crypto::rand() + }; + + lws::account full_account = lws::db::test::make_account(account, view); + full_account.updated(last_block.id); + EXPECT(add_out(full_account, last_block.id, 500)); + + const std::vector outs = full_account.outputs(); + EXPECT(outs.size() == 1); + + const auto updated = db.update(last_block.id, chain, {std::addressof(full_account), 1}); + EXPECT(!updated.has_error()); + EXPECT(updated->first == 1); + EXPECT(updated->second.size() == 3); + + for (unsigned i = 0; i < 3; ++i) + { + EXPECT(updated->second[i].key.user == lws::db::account_id(1)); + EXPECT(updated->second[i].key.type == lws::db::webhook_type::tx_confirmation); + EXPECT(updated->second[i].value.first.payment_id == 500); + EXPECT(updated->second[i].value.first.event_id == id); + EXPECT(updated->second[i].value.second.url == "http://the_url"); + EXPECT(updated->second[i].value.second.token == "the_token"); + EXPECT(updated->second[i].value.second.confirmations == i + 1); + + EXPECT(updated->second[i].tx_info.link == outs[0].link); + EXPECT(updated->second[i].tx_info.spend_meta.id == outs[0].spend_meta.id); + EXPECT(updated->second[i].tx_info.pub == outs[0].pub); + EXPECT(updated->second[i].tx_info.payment_id.short_ == outs[0].payment_id.short_); + } + } + } +} diff --git a/tests/unit/rpc/CMakeLists.txt b/tests/unit/rpc/CMakeLists.txt new file mode 100644 index 0000000..f8c2e40 --- /dev/null +++ b/tests/unit/rpc/CMakeLists.txt @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2023, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +add_library(monero-lws-unit-rpc OBJECT admin.test.cpp) +target_link_libraries( + monero-lws-unit-rpc + monero-lws-unit-db + monero-lws-unit-framework + monero-lws-common + monero-lws-db + monero-lws-rpc + monero-lws-wire-json + monero::libraries +) +#add_test(monero-lws-unit) diff --git a/tests/unit/rpc/admin.test.cpp b/tests/unit/rpc/admin.test.cpp new file mode 100644 index 0000000..a9a5dfe --- /dev/null +++ b/tests/unit/rpc/admin.test.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "framework.test.h" + +#include +#include "db/storage.test.h" +#include "db/string.h" +#include "error.h" +#include "hex.h" // monero/contrib/epee/include +#include "rpc/admin.h" +#include "wire/json.h" + +namespace +{ + constexpr const char address_str[] = + u8"42ui2zRV3KBKgHPnQHDZu7WFc397XmhEjL9e6UnSpyHiKh4vydo7atvaQDSDKYPoCb51GQZc7hZZvDrJM7JCyuYqHHbshVn"; + constexpr const char view_str[] = + u8"9ec001644f8d79ecb368083e48e7efb5a48b3563c9a78ba497874fd58285330d"; + + template + expect call_endpoint(lws::db::storage disk, std::string json) + { + using request_type = typename T::request; + expect req = wire::json::from_bytes(std::move(json)); + if (!req) + return req.error(); + wire::json_slice_writer out{}; + MONERO_CHECK(T{}(out, std::move(disk), std::move(*req))); + return out.take_bytes(); + } +} + +LWS_CASE("rpc::admin") +{ + lws::db::account_address account = MONERO_UNWRAP(lws::db::address_string(address_str)); + crypto::secret_key view{}; + EXPECT(epee::from_hex::to_buffer(epee::as_mut_byte_span(unwrap(unwrap(view))), view_str)); + + SETUP("One Account One Webhook Database") + { + lws::db::test::cleanup_db on_scope_exit{}; + lws::db::storage db = lws::db::test::get_fresh_db(); + const lws::db::block_info last_block = + MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_last_block()); + MONERO_UNWRAP(db.add_account(account, view)); + + boost::uuids::uuid id{}; + epee::byte_slice id_str{}; + expect result{lws::error::configuration}; + { + std::string add_json_str{}; + add_json_str.append(u8"{\"url\":\"http://the_url\", \"token\":\"the_token\","); + add_json_str.append(u8"\"address\":\"").append(address_str).append(u8"\","); + add_json_str.append(u8"\"payment_id\":\"deadbeefdeadbeef\","); + add_json_str.append(u8"\"type\":\"tx-confirmation\",\"confirmations\":3}"); + result = call_endpoint(db.clone(), std::move(add_json_str)); + EXPECT(!result.has_error()); + } + + { + static constexpr const char begin[] = + u8"{\"payment_id\":\"deadbeefdeadbeef\",\"event_id\":\""; + epee::byte_slice begin_ = result->take_slice(sizeof(begin) - 1); + EXPECT(boost::range::equal(std::string{begin}, begin_)); + } + { + id_str = result->take_slice(32); + const boost::string_ref id_hex{ + reinterpret_cast(id_str.data()), id_str.size() + }; + EXPECT(epee::from_hex::to_buffer(epee::as_mut_byte_span(id), id_hex)); + } + + SECTION("webhook_add") + { + static constexpr const char end[] = + u8"\",\"token\":\"the_token\",\"confirmations\":3,\"url\":\"http://the_url\"}"; + EXPECT(boost::range::equal(std::string{end}, *result)); + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).size() == 1); + } + + SECTION("webhook_delete_uuid") + { + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).size() == 1); + std::string delete_json_str{}; + delete_json_str.append(u8"{\"addresses\":[\""); + delete_json_str.append(address_str); + delete_json_str.append(u8"\"]}"); + + expect result2 = + call_endpoint(db.clone(), std::move(delete_json_str)); + EXPECT(!result2.has_error()); + EXPECT(boost::range::equal(std::string{u8"{}"}, *result2)); + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).empty()); + } + + SECTION("webhook_delete_uuid") + { + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).size() == 1); + std::string delete_json_str{}; + delete_json_str.append(u8"{\"event_ids\":[\""); + delete_json_str.append(reinterpret_cast(id_str.data()), id_str.size()); + delete_json_str.append(u8"\"]}"); + + expect result2 = + call_endpoint(db.clone(), std::move(delete_json_str)); + EXPECT(!result2.has_error()); + EXPECT(boost::range::equal(std::string{u8"{}"}, *result2)); + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_webhooks()).empty()); + } + + SECTION("webhook_list") + { + wire::json_slice_writer out{}; + EXPECT(lws::rpc::webhook_list(out, db.clone())); + expect bytes = out.take_bytes(); + EXPECT(!bytes.has_error()); + + { + static constexpr const char begin[] = + u8"{\"webhooks\":[{\"key\":{\"user\":1,\"type\":\"tx-confirmation\"}" + ",\"value\":[{\"payment_id\":\"deadbeefdeadbeef\",\"event_id\":\""; + epee::byte_slice begin_ = bytes->take_slice(sizeof(begin) - 1); + EXPECT(boost::range::equal(std::string{begin}, begin_)); + } + { + boost::uuids::uuid id_{}; + epee::byte_slice id_str_ = bytes->take_slice(32); + const boost::string_ref id_hex{ + reinterpret_cast(id_str_.data()), id_str_.size() + }; + EXPECT(epee::from_hex::to_buffer(epee::as_mut_byte_span(id_), id_hex)); + EXPECT(id_ == id); + } + { + static constexpr const char end[] = + u8"\",\"token\":\"the_token\",\"confirmations\":3,\"url\":\"http://the_url\"}]}]}"; + EXPECT(boost::range::equal(std::string{end}, *bytes)); + } + } + } +}