diff --git a/CMakeLists.txt b/CMakeLists.txt index 5336d84..5daea52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ project(monero-lws) enable_language(CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +add_definitions(-DBOOST_UUID_DISABLE_ALIGNMENT) # This restores UUID's std::has_unique_object_representations property option(BUILD_TESTS "Build Tests" OFF) option(WITH_RMQ "Build with RMQ publish support" OFF) diff --git a/src/db/storage.cpp b/src/db/storage.cpp index da4b590..6d45617 100644 --- a/src/db/storage.cpp +++ b/src/db/storage.cpp @@ -49,6 +49,7 @@ #include "error.h" #include "hex.h" #include "lmdb/lws_database.h" +#include "lmdb/lws_table.h" #include "lmdb/error.h" #include "lmdb/key_stream.h" #include "lmdb/msgpack_table.h" @@ -333,7 +334,7 @@ namespace db constexpr const lmdb::msgpack_table webhooks{ "webhooks_by_account_id,payment_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less }; - constexpr const lmdb::basic_table events_by_account_id{ + constexpr const lws_lmdb::basic_table events_by_account_id{ "webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less }; constexpr const lmdb::msgpack_table subaddress_ranges{ diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt index 3d8949e..9e1731b 100644 --- a/src/lmdb/CMakeLists.txt +++ b/src/lmdb/CMakeLists.txt @@ -27,7 +27,7 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(monero-lws-lmdb_sources lws_database.cpp) -set(monero-lws-lmdb_headers lws_database.h msgpack_table.h) +set(monero-lws-lmdb_headers lws_database.h lws_key_stream.h lws_table.h lws_value_stream.h msgpack_table.h) add_library(monero-lws-lmdb ${monero-lws-lmdb_sources} ${monero-lws-lmdb_headers}) target_include_directories(monero-lws-lmdb PUBLIC "${LMDB_INCLUDE}") diff --git a/src/lmdb/lws_key_stream.h b/src/lmdb/lws_key_stream.h new file mode 100644 index 0000000..36b5050 --- /dev/null +++ b/src/lmdb/lws_key_stream.h @@ -0,0 +1,267 @@ +// Copyright (c) 2018-2024, The Monero Project + +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "lmdb/lws_value_stream.h" +#include "span.h" + +namespace lws_lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template + class key_iterator + { + MDB_cursor* cur; + epee::span key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + + static_assert(std::is_trivially_copyable(), "key is not memcpyable"); + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template + value_iterator make_value_iterator() const + { + static_assert(std::is_same(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template + boost::iterator_range> make_value_range() const + { + return {make_value_iterator(), value_iterator{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template + class key_stream + { + std::unique_ptr cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range> make_range() const + { + return {make_iterator(), key_iterator{}}; + } + }; + + template + inline + bool operator==(key_iterator const& lhs, key_iterator const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template + inline + bool operator!=(key_iterator const& lhs, key_iterator const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lws_lmdb + diff --git a/src/lmdb/lws_table.h b/src/lmdb/lws_table.h new file mode 100644 index 0000000..cd1a97d --- /dev/null +++ b/src/lmdb/lws_table.h @@ -0,0 +1,109 @@ +#pragma once + +#include + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/lws_key_stream.h" +#include "lmdb/table.h" +#include "lmdb/util.h" +#include "lmdb/lws_value_stream.h" + +namespace lws_lmdb +{ + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template + struct basic_table : lmdb::table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : lmdb::table{name, compute_flags(flags), &lmdb::less>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template + static expect get_value(MDB_val value) noexcept + { + static_assert(std::is_same(), "bad MONERO_FIELD?"); + static_assert(std::is_trivially_copyable(), "F must be memcpyable"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template + expect> + static get_key_stream(std::unique_ptr cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template + expect> + static get_value_stream(K const& key, std::unique_ptr cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream{std::move(cur)}; + } + }; +} // lws_lmdb + diff --git a/src/lmdb/lws_value_stream.h b/src/lmdb/lws_value_stream.h new file mode 100644 index 0000000..8be7475 --- /dev/null +++ b/src/lmdb/lws_value_stream.h @@ -0,0 +1,262 @@ +// Copyright (c) 2018-2024, The Monero Project + +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lws_lmdb +{ + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template + class value_iterator + { + MDB_cursor* cur; + epee::span values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template + G get_value() const noexcept + { + static_assert(std::is_same(), "bad MONERO_FIELD usage?"); + static_assert(std::is_trivially_copyable(), "value type must be memcpyable"); + static_assert(std::is_trivially_copyable(), "field type must be memcpyable"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template + class value_stream + { + std::unique_ptr cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template + value_iterator make_iterator() const + { + static_assert(std::is_same(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template + boost::iterator_range> make_range() const + { + return {make_iterator(), value_iterator{}}; + } + }; + + template + inline + bool operator==(value_iterator const& lhs, value_iterator const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template + inline + bool operator!=(value_iterator const& lhs, value_iterator const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lws_lmdb + diff --git a/src/lmdb/msgpack_table.h b/src/lmdb/msgpack_table.h index 4c70abe..5b20e90 100644 --- a/src/lmdb/msgpack_table.h +++ b/src/lmdb/msgpack_table.h @@ -29,6 +29,7 @@ namespace lmdb return {lmdb::error(MDB_BAD_VALSIZE)}; key_type out; + static_assert(std::is_trivially_copyable(), "key_type is not memcpyable"); std::memcpy(std::addressof(out), static_cast(key.mv_data), sizeof(out)); return out; } @@ -66,7 +67,7 @@ namespace lmdb static expect get_fixed_value(MDB_val value) noexcept { static_assert(std::is_same(), "bad MONERO_FIELD?"); - static_assert(std::is_pod(), "F must be POD"); + static_assert(std::is_trivially_copyable(), "F must be memcpyable"); static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); if (value.mv_size < sizeof(U)) @@ -79,6 +80,11 @@ namespace lmdb static expect get_value(MDB_val value) noexcept { + static_assert( + std::is_trivially_copyable(), + "fixed_value_type must be memcpyable" + ); + if (value.mv_size < sizeof(fixed_value_type)) return {lmdb::error(MDB_BAD_VALSIZE)}; std::pair out; @@ -134,4 +140,4 @@ namespace lmdb return out; } }; -} // lmdb +} // lws_lmdb diff --git a/tests/unit/rest.test.cpp b/tests/unit/rest.test.cpp index 8e421f4..ed09a4c 100644 --- a/tests/unit/rest.test.cpp +++ b/tests/unit/rest.test.cpp @@ -276,7 +276,7 @@ LWS_CASE("rest_server") "\"fee\":\"100\"," "\"unlock_time\":4670," "\"height\":4000," - "\"payment_id\":\"" + epee::to_hex::string(epee::as_byte_span(payment_id_)) + "\"," + "\"payment_id\":\"" + epee::to_hex::string(epee::as_byte_span(payment_id_.long_)) + "\"," "\"coinbase\":true," "\"mempool\":false," "\"mixin\":16," @@ -425,7 +425,7 @@ LWS_CASE("rest_server") "\"fee\":\"100\"," "\"unlock_time\":4670," "\"height\":4000," - "\"payment_id\":\"" + epee::to_hex::string(epee::as_byte_span(payment_id_)) + "\"," + "\"payment_id\":\"" + epee::to_hex::string(epee::as_byte_span(payment_id_.long_)) + "\"," "\"coinbase\":true," "\"mempool\":false," "\"mixin\":16,"