From 42f9f15a45aef4f2aa91be86b4edeebdffcc6863 Mon Sep 17 00:00:00 2001
From: Lee *!* Clagett <code@leeclagett.com>
Date: Fri, 9 Aug 2024 19:59:55 -0400
Subject: [PATCH] Pull in fixes to lmdb::database from monero PR
---
src/CMakeLists.txt | 1 +
src/db/CMakeLists.txt | 2 +-
src/db/storage.cpp | 10 +--
src/lmdb/CMakeLists.txt | 34 ++++++++
src/lmdb/lws_database.cpp | 173 ++++++++++++++++++++++++++++++++++++++
src/lmdb/lws_database.h | 142 +++++++++++++++++++++++++++++++
6 files changed, 356 insertions(+), 6 deletions(-)
create mode 100644 src/lmdb/CMakeLists.txt
create mode 100644 src/lmdb/lws_database.cpp
create mode 100644 src/lmdb/lws_database.h
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 81436eb..019fb6c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -28,6 +28,7 @@
include_directories(.)
+add_subdirectory(lmdb)
add_subdirectory(wire)
add_subdirectory(db)
add_subdirectory(rpc)
diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt
index 2030ac2..48a947e 100644
--- a/src/db/CMakeLists.txt
+++ b/src/db/CMakeLists.txt
@@ -31,4 +31,4 @@ set(monero-lws-db_headers account.h data.h fwd.h storage.h string.h)
add_library(monero-lws-db ${monero-lws-db_sources} ${monero-lws-db_headers})
target_include_directories(monero-lws-db PUBLIC "${LMDB_INCLUDE}")
-target_link_libraries(monero-lws-db monero::libraries monero-lws-common monero-lws-wire-msgpack ${LMDB_LIB_PATH})
+target_link_libraries(monero-lws-db monero::libraries monero-lws-common monero-lws-lmdb monero-lws-wire-msgpack ${LMDB_LIB_PATH})
diff --git a/src/db/storage.cpp b/src/db/storage.cpp
index 8e09e0a..da4b590 100644
--- a/src/db/storage.cpp
+++ b/src/db/storage.cpp
@@ -48,7 +48,7 @@
#include "db/string.h"
#include "error.h"
#include "hex.h"
-#include "lmdb/database.h"
+#include "lmdb/lws_database.h"
#include "lmdb/error.h"
#include "lmdb/key_stream.h"
#include "lmdb/msgpack_table.h"
@@ -653,7 +653,7 @@ namespace db
}
} // anonymous
- struct storage_internal : lmdb::database
+ struct storage_internal : lws_lmdb::database
{
struct tables_
{
@@ -674,8 +674,8 @@ namespace db
const unsigned create_queue_max;
- explicit storage_internal(lmdb::environment env, unsigned create_queue_max)
- : lmdb::database(std::move(env)), tables{}, create_queue_max(create_queue_max)
+ explicit storage_internal(lws_lmdb::environment env, unsigned create_queue_max)
+ : lws_lmdb::database(std::move(env)), tables{}, create_queue_max(create_queue_max)
{
lmdb::write_txn txn = this->create_write_txn().value();
assert(txn != nullptr);
@@ -1403,7 +1403,7 @@ namespace db
{
return {
std::make_shared<storage_internal>(
- MONERO_UNWRAP(lmdb::open_environment(path, 20)), create_queue_max
+ MONERO_UNWRAP(lws_lmdb::open_environment(path, 20)), create_queue_max
)
};
}
diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt
new file mode 100644
index 0000000..3d8949e
--- /dev/null
+++ b/src/lmdb/CMakeLists.txt
@@ -0,0 +1,34 @@
+# Copyright (c) 2024, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+# conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+# of conditions and the following disclaimer in the documentation and/or other
+# materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+# used to endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-lmdb_sources lws_database.cpp)
+set(monero-lws-lmdb_headers lws_database.h msgpack_table.h)
+
+add_library(monero-lws-lmdb ${monero-lws-lmdb_sources} ${monero-lws-lmdb_headers})
+target_include_directories(monero-lws-lmdb PUBLIC "${LMDB_INCLUDE}")
+target_link_libraries(monero-lws-lmdb PRIVATE monero::libraries)
diff --git a/src/lmdb/lws_database.cpp b/src/lmdb/lws_database.cpp
new file mode 100644
index 0000000..85c3e5e
--- /dev/null
+++ b/src/lmdb/lws_database.cpp
@@ -0,0 +1,173 @@
+// Copyright (c) 2014-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+// conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+// of conditions and the following disclaimer in the documentation and/or other
+// materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+// used to endorse or promote products derived from this software without specific
+// prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "lws_database.h"
+#include "lmdb/error.h"
+#include "lmdb/util.h"
+
+#ifdef _WIN32
+namespace
+{
+ constexpr const mdb_mode_t open_flags = 0;
+}
+#else
+#include <sys/stat.h>
+
+namespace
+{
+ constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR);
+}
+#endif
+
+namespace lws_lmdb
+{
+ namespace
+ {
+ constexpr const mdb_size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB
+ void acquire_context(context& ctx) noexcept
+ {
+ while (ctx.lock.test_and_set());
+ ++(ctx.active);
+ ctx.lock.clear();
+ }
+
+ void release_context(context& ctx) noexcept
+ {
+ --(ctx.active);
+ }
+ }
+
+ expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept
+ {
+ MONERO_PRECOND(path != nullptr);
+
+ MDB_env* obj = nullptr;
+ MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj)));
+ environment out{obj};
+
+ MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs));
+ MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags));
+ return {std::move(out)};
+ }
+
+ expect<lmdb::write_txn> database::do_create_txn(unsigned int flags) noexcept
+ {
+ MONERO_PRECOND(handle() != nullptr);
+
+ for (unsigned attempts = 0; attempts < 3; ++attempts)
+ {
+ acquire_context(ctx);
+
+ MDB_txn* txn = nullptr;
+ const int err =
+ mdb_txn_begin(handle(), nullptr, flags, &txn);
+ if (!err && txn != nullptr)
+ return lmdb::write_txn{txn};
+
+ release_context(ctx);
+ if (err != MDB_MAP_RESIZED)
+ return {lmdb::error(err)};
+ MONERO_CHECK(this->resize());
+ }
+ return {lmdb::error(MDB_MAP_RESIZED)};
+ }
+
+ database::database(environment env)
+ : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT}
+ {
+ if (handle())
+ {
+ const int err = mdb_env_set_userctx(handle(), std::addressof(ctx));
+ if (err)
+ MONERO_THROW(lmdb::error(err), "Failed to set user context");
+ }
+ }
+
+ database::~database() noexcept
+ {
+ while (ctx.active);
+ }
+
+ expect<void> database::resize() noexcept
+ {
+ MONERO_PRECOND(handle() != nullptr);
+
+ while (ctx.lock.test_and_set());
+ while (ctx.active);
+
+ MDB_envinfo info{};
+ MONERO_LMDB_CHECK(mdb_env_info(handle(), &info));
+
+ const mdb_size_t resize = std::min(info.me_mapsize, max_resize);
+ const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize);
+ ctx.lock.clear();
+ if (err)
+ return {lmdb::error(err)};
+ return success();
+ }
+
+ expect<lmdb::read_txn> database::create_read_txn(lmdb::suspended_txn txn) noexcept
+ {
+ if (txn)
+ {
+ acquire_context(ctx);
+ const int err = mdb_txn_renew(txn.get());
+ if (err)
+ {
+ release_context(ctx);
+ return {lmdb::error(err)};
+ }
+ return lmdb::read_txn{txn.release()};
+ }
+ auto new_txn = do_create_txn(MDB_RDONLY);
+ if (new_txn)
+ return lmdb::read_txn{new_txn->release()};
+ return new_txn.error();
+ }
+
+ expect<lmdb::suspended_txn> database::reset_txn(lmdb::read_txn txn) noexcept
+ {
+ MONERO_PRECOND(txn != nullptr);
+ mdb_txn_reset(txn.get());
+ release_context(ctx);
+ return lmdb::suspended_txn{txn.release()};
+ }
+
+ expect<lmdb::write_txn> database::create_write_txn() noexcept
+ {
+ return do_create_txn(0);
+ }
+
+ expect<void> database::commit(lmdb::write_txn txn) noexcept
+ {
+ MONERO_PRECOND(txn != nullptr);
+ const int err = mdb_txn_commit(txn.release());
+ release_context(ctx);
+ if (err)
+ return {lmdb::error(err)};
+ return success();
+ }
+} // lws_lmdb
diff --git a/src/lmdb/lws_database.h b/src/lmdb/lws_database.h
new file mode 100644
index 0000000..47da73f
--- /dev/null
+++ b/src/lmdb/lws_database.h
@@ -0,0 +1,142 @@
+// Copyright (c) 2018-2023, The Monero Project
+
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+// conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+// of conditions and the following disclaimer in the documentation and/or other
+// materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+// used to endorse or promote products derived from this software without specific
+// prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <atomic>
+#include <cstddef>
+#include <lmdb.h>
+#include <memory>
+#include <type_traits>
+
+#include "common/expect.h"
+#include "lmdb/error.h"
+#include "lmdb/transaction.h"
+
+namespace lws_lmdb
+{
+ //! Closes LMDB environment handle.
+ struct close_env
+ {
+ void operator()(MDB_env* ptr) const noexcept
+ {
+ if (ptr)
+ mdb_env_close(ptr);
+ }
+ };
+
+ using environment = std::unique_ptr<MDB_env, close_env>;
+
+ //! \return LMDB environment at `path` with a max of `max_dbs` tables.
+ expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept;
+
+ //! Context given to LMDB.
+ struct context
+ {
+ std::atomic<std::size_t> active;
+ std::atomic_flag lock;
+ };
+
+ //! Manages a LMDB environment for safe memory-map resizing. Thread-safe.
+ class database
+ {
+ environment env;
+ context ctx;
+
+ //! \return The LMDB environment associated with the object.
+ MDB_env* handle() const noexcept { return env.get(); }
+
+ expect<::lmdb::write_txn> do_create_txn(unsigned int flags) noexcept;
+
+ public:
+ database(environment env);
+
+ database(database&&) = delete;
+ database(database const&) = delete;
+
+ virtual ~database() noexcept;
+
+ database& operator=(database&&) = delete;
+ database& operator=(database const&) = delete;
+
+ /*!
+ Resize the memory map for the LMDB environment. Will block until
+ all reads/writes on the environment complete.
+ */
+ expect<void> resize() noexcept;
+
+ //! \return A read only LMDB transaction, reusing `txn` if provided.
+ expect<lmdb::read_txn> create_read_txn(lmdb::suspended_txn txn = nullptr) noexcept;
+
+ //! \return `txn` after releasing context.
+ expect<lmdb::suspended_txn> reset_txn(lmdb::read_txn txn) noexcept;
+
+ //! \return A read-write LMDB transaction.
+ expect<lmdb::write_txn> create_write_txn() noexcept;
+
+ //! Commit the read-write transaction.
+ expect<void> commit(lmdb::write_txn txn) noexcept;
+
+ /*!
+ Create a write transaction, pass it to `f`, then try to commit
+ the write if `f` succeeds.
+
+ \tparam F must be callable with signature `expect<T>(MDB_txn&)`.
+ \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`.
+
+ \return The result of calling `f`.
+ */
+ template<typename F>
+ typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3)
+ {
+ for (unsigned i = 0; i < attempts; ++i)
+ {
+ expect<lmdb::write_txn> txn = create_write_txn();
+ if (!txn)
+ return txn.error();
+
+ MONERO_PRECOND(*txn != nullptr);
+ const auto wrote = f(*(*txn));
+ if (wrote)
+ {
+ const auto committed = commit(std::move(*txn));
+ if (committed)
+ return wrote;
+ if (committed != lmdb::error(MDB_MAP_FULL))
+ return committed.error();
+ }
+ else if (wrote != lmdb::error(MDB_MAP_FULL))
+ return wrote;
+
+ txn->reset();
+ MONERO_CHECK(this->resize());
+ }
+ return {lmdb::error(MDB_MAP_FULL)};
+ }
+ };
+} // lws_lmdb
+