Pull in fixes to lmdb::database from monero PR (#130)

src/CMakeLists.txt

@@ -28,6 +28,7 @@
 
 include_directories(.)
 
+add_subdirectory(lmdb)
 add_subdirectory(wire)
 add_subdirectory(db)
 add_subdirectory(rpc)

src/db/CMakeLists.txt

@@ -31,4 +31,4 @@ set(monero-lws-db_headers account.h data.h fwd.h storage.h string.h)
 
 add_library(monero-lws-db ${monero-lws-db_sources} ${monero-lws-db_headers})
 target_include_directories(monero-lws-db PUBLIC "${LMDB_INCLUDE}")
-target_link_libraries(monero-lws-db monero::libraries monero-lws-common monero-lws-wire-msgpack ${LMDB_LIB_PATH})
+target_link_libraries(monero-lws-db monero::libraries monero-lws-common monero-lws-lmdb monero-lws-wire-msgpack ${LMDB_LIB_PATH})

src/db/storage.cpp

@@ -48,7 +48,7 @@
 #include "db/string.h"
 #include "error.h"
 #include "hex.h"
-#include "lmdb/database.h"
+#include "lmdb/lws_database.h"
 #include "lmdb/error.h"
 #include "lmdb/key_stream.h"
 #include "lmdb/msgpack_table.h"
@@ -653,7 +653,7 @@ namespace db
     }
   } // anonymous
 
-  struct storage_internal : lmdb::database
+  struct storage_internal : lws_lmdb::database
   {
     struct tables_
     {
@@ -674,8 +674,8 @@ namespace db
 
     const unsigned create_queue_max;
 
-    explicit storage_internal(lmdb::environment env, unsigned create_queue_max)
+    explicit storage_internal(lws_lmdb::environment env, unsigned create_queue_max)
-      : lmdb::database(std::move(env)), tables{}, create_queue_max(create_queue_max)
+      : lws_lmdb::database(std::move(env)), tables{}, create_queue_max(create_queue_max)
     {
       lmdb::write_txn txn = this->create_write_txn().value();
       assert(txn != nullptr);
@@ -1403,7 +1403,7 @@ namespace db
   {
     return {
       std::make_shared<storage_internal>(
-        MONERO_UNWRAP(lmdb::open_environment(path, 20)), create_queue_max
+        MONERO_UNWRAP(lws_lmdb::open_environment(path, 20)), create_queue_max
       )
     };
   }

src/lmdb/CMakeLists.txt

@@ -0,0 +1,34 @@
+# Copyright (c) 2024, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-lmdb_sources lws_database.cpp)
+set(monero-lws-lmdb_headers lws_database.h msgpack_table.h)
+
+add_library(monero-lws-lmdb ${monero-lws-lmdb_sources} ${monero-lws-lmdb_headers})
+target_include_directories(monero-lws-lmdb PUBLIC "${LMDB_INCLUDE}")
+target_link_libraries(monero-lws-lmdb PRIVATE monero::libraries)

src/lmdb/lws_database.cpp

@@ -0,0 +1,173 @@
+// Copyright (c) 2014-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "lws_database.h"
+#include "lmdb/error.h"
+#include "lmdb/util.h"
+
+#ifdef _WIN32
+namespace
+{
+    constexpr const mdb_mode_t open_flags = 0;
+}
+#else
+#include <sys/stat.h>
+
+namespace
+{
+    constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR);
+}
+#endif
+
+namespace lws_lmdb
+{
+    namespace
+    {
+        constexpr const mdb_size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB
+        void acquire_context(context& ctx) noexcept
+        {
+            while (ctx.lock.test_and_set());
+            ++(ctx.active);
+            ctx.lock.clear();
+        }
+
+        void release_context(context& ctx) noexcept
+        {
+            --(ctx.active);
+        }
+    }
+
+    expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept
+    {
+        MONERO_PRECOND(path != nullptr);
+
+        MDB_env* obj = nullptr;
+        MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj)));
+        environment out{obj};
+
+        MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs));
+        MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags));
+        return {std::move(out)};
+    }
+
+    expect<lmdb::write_txn> database::do_create_txn(unsigned int flags) noexcept
+    {
+        MONERO_PRECOND(handle() != nullptr);
+
+        for (unsigned attempts = 0; attempts < 3; ++attempts)
+        {
+            acquire_context(ctx);
+
+            MDB_txn* txn = nullptr;
+            const int err =
+                mdb_txn_begin(handle(), nullptr, flags, &txn);
+            if (!err && txn != nullptr)
+                return lmdb::write_txn{txn};
+
+            release_context(ctx);
+            if (err != MDB_MAP_RESIZED)
+                return {lmdb::error(err)};
+            MONERO_CHECK(this->resize());
+        }
+        return {lmdb::error(MDB_MAP_RESIZED)};
+    }
+
+    database::database(environment env)
+      : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT}
+    {
+        if (handle())
+        {
+            const int err = mdb_env_set_userctx(handle(), std::addressof(ctx));
+            if (err)
+                MONERO_THROW(lmdb::error(err), "Failed to set user context");
+        }
+    }
+
+    database::~database() noexcept
+    {
+        while (ctx.active);
+    }
+
+    expect<void> database::resize() noexcept
+    {
+        MONERO_PRECOND(handle() != nullptr);
+
+        while (ctx.lock.test_and_set());
+        while (ctx.active);
+
+        MDB_envinfo info{};
+        MONERO_LMDB_CHECK(mdb_env_info(handle(), &info));
+
+        const mdb_size_t resize = std::min(info.me_mapsize, max_resize);
+        const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize);
+        ctx.lock.clear();
+        if (err)
+            return {lmdb::error(err)};
+        return success();
+    }
+
+    expect<lmdb::read_txn> database::create_read_txn(lmdb::suspended_txn txn) noexcept
+    {
+        if (txn)
+        {
+            acquire_context(ctx);
+            const int err = mdb_txn_renew(txn.get());
+            if (err)
+            {
+                release_context(ctx);
+                return {lmdb::error(err)};
+            }
+            return lmdb::read_txn{txn.release()};
+        }
+        auto new_txn = do_create_txn(MDB_RDONLY);
+        if (new_txn)
+            return lmdb::read_txn{new_txn->release()};
+        return new_txn.error();
+    }
+
+    expect<lmdb::suspended_txn> database::reset_txn(lmdb::read_txn txn) noexcept
+    {
+        MONERO_PRECOND(txn != nullptr);
+        mdb_txn_reset(txn.get());
+        release_context(ctx);
+        return lmdb::suspended_txn{txn.release()};
+    }
+
+    expect<lmdb::write_txn> database::create_write_txn() noexcept
+    {
+        return do_create_txn(0);
+    }
+
+    expect<void> database::commit(lmdb::write_txn txn) noexcept
+    {
+        MONERO_PRECOND(txn != nullptr);
+        const int err = mdb_txn_commit(txn.release());
+        release_context(ctx);
+        if (err)
+          return {lmdb::error(err)};
+        return success();
+    }
+} // lws_lmdb

src/lmdb/lws_database.h

@@ -0,0 +1,142 @@
+// Copyright (c) 2018-2023, The Monero Project
+
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <atomic>
+#include <cstddef>
+#include <lmdb.h>
+#include <memory>
+#include <type_traits>
+
+#include "common/expect.h"
+#include "lmdb/error.h"
+#include "lmdb/transaction.h"
+
+namespace lws_lmdb
+{
+    //! Closes LMDB environment handle.
+    struct close_env
+    {
+        void operator()(MDB_env* ptr) const noexcept
+        {
+            if (ptr)
+                mdb_env_close(ptr);
+        }
+    };
+
+    using environment = std::unique_ptr<MDB_env, close_env>;
+
+    //! \return LMDB environment at `path` with a max of `max_dbs` tables.
+    expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept;
+
+    //! Context given to LMDB.
+    struct context
+    {
+        std::atomic<std::size_t> active;
+        std::atomic_flag lock;
+    };
+
+    //! Manages a LMDB environment for safe memory-map resizing. Thread-safe.
+    class database
+    {
+        environment env;
+        context ctx;
+
+        //! \return The LMDB environment associated with the object.
+        MDB_env* handle() const noexcept { return env.get(); }
+
+        expect<::lmdb::write_txn> do_create_txn(unsigned int flags) noexcept;
+
+    public: 
+        database(environment env);
+
+        database(database&&) = delete;
+        database(database const&) = delete;
+
+        virtual ~database() noexcept;
+
+        database& operator=(database&&) = delete;
+        database& operator=(database const&) = delete;
+
+        /*!
+            Resize the memory map for the LMDB environment. Will block until
+            all reads/writes on the environment complete.
+        */
+        expect<void> resize() noexcept;
+
+        //! \return A read only LMDB transaction, reusing `txn` if provided.
+        expect<lmdb::read_txn> create_read_txn(lmdb::suspended_txn txn = nullptr) noexcept;
+
+        //! \return `txn` after releasing context.
+        expect<lmdb::suspended_txn> reset_txn(lmdb::read_txn txn) noexcept;
+
+        //! \return A read-write LMDB transaction.
+        expect<lmdb::write_txn> create_write_txn() noexcept;
+
+        //! Commit the read-write transaction.
+        expect<void> commit(lmdb::write_txn txn) noexcept;
+
+        /*!
+            Create a write transaction, pass it to `f`, then try to commit
+            the write if `f` succeeds.
+
+            \tparam F must be callable with signature `expect<T>(MDB_txn&)`.
+            \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`.
+
+            \return The result of calling `f`.
+        */
+        template<typename F>
+        typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3)
+        {
+            for (unsigned i = 0; i < attempts; ++i)
+            {
+                expect<lmdb::write_txn> txn = create_write_txn();
+                if (!txn)
+                    return txn.error();
+
+                MONERO_PRECOND(*txn != nullptr);
+                const auto wrote = f(*(*txn));
+                if (wrote)
+                {
+                    const auto committed = commit(std::move(*txn));
+                    if (committed)
+                      return wrote;
+                    if (committed != lmdb::error(MDB_MAP_FULL))
+                      return committed.error();
+                }
+                else if (wrote != lmdb::error(MDB_MAP_FULL))
+                    return wrote;
+
+                txn->reset();
+                MONERO_CHECK(this->resize());
+            }
+            return {lmdb::error(MDB_MAP_FULL)};
+        }
+    };
+} // lws_lmdb
+