Merge pull request #1794 from SChernykh/dev

More robust 1 GB pages handling

src/backend/cpu/CpuWorker.cpp

@@ -36,6 +36,7 @@
 #include "crypto/common/Nonce.h"
 #include "crypto/common/VirtualMemory.h"
 #include "crypto/rx/Rx.h"
+#include "crypto/rx/RxDataset.h"
 #include "crypto/rx/RxVm.h"
 #include "net/JobResults.h"
 
@@ -118,7 +119,9 @@ void xmrig::CpuWorker<N>::allocateRandomX_VM()
     }
 
     if (!m_vm) {
-        m_vm = RxVm::create(dataset, m_memory->scratchpad(), !m_hwAES, m_assembly, m_node);
+        // Try to allocate scratchpad from dataset's 1 GB huge pages, if normal huge pages are not available
+        uint8_t* scratchpad = m_memory->isHugePages() ? m_memory->scratchpad() : dataset->tryAllocateScrathpad();
+        m_vm = RxVm::create(dataset, scratchpad ? scratchpad : m_memory->scratchpad(), !m_hwAES, m_assembly, m_node);
     }
 }
 #endif

src/crypto/common/VirtualMemory.cpp

@@ -51,6 +51,7 @@ static std::mutex mutex;
 
 xmrig::VirtualMemory::VirtualMemory(size_t size, bool hugePages, bool oneGbPages, bool usePool, uint32_t node, size_t alignSize) :
     m_size(align(size)),
+    m_capacity(m_size),
     m_node(node)
 {
     if (usePool) {
@@ -69,6 +70,7 @@ xmrig::VirtualMemory::VirtualMemory(size_t size, bool hugePages, bool oneGbPages
     }
 
     if (oneGbPages && allocateOneGbPagesMemory()) {
+        m_capacity = align(size, 1ULL << 30);
         return;
     }
 

src/crypto/common/VirtualMemory.h

@@ -52,6 +52,7 @@ public:
     inline bool isHugePages() const     { return m_flags.test(FLAG_HUGEPAGES); }
     inline bool isOneGbPages() const    { return m_flags.test(FLAG_1GB_PAGES); }
     inline size_t size() const          { return m_size; }
+    inline size_t capacity() const      { return m_capacity; }
     inline uint8_t *raw() const         { return m_scratchpad; }
     inline uint8_t *scratchpad() const  { return m_scratchpad; }
 
@@ -88,6 +89,7 @@ private:
     void freeLargePagesMemory();
 
     const size_t m_size;
+    size_t m_capacity;
     const uint32_t m_node;
     std::bitset<FLAG_MAX> m_flags;
     uint8_t *m_scratchpad = nullptr;

src/crypto/common/VirtualMemory_unix.cpp

@@ -82,7 +82,17 @@ void *xmrig::VirtualMemory::allocateLargePagesMemory(size_t size)
 #   elif defined(__FreeBSD__)
     void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER | MAP_PREFAULT_READ, -1, 0);
 #   else
-    void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, 0, 0);
+
+#   if defined(MAP_HUGE_2MB)
+    constexpr int flag_2mb = MAP_HUGE_2MB;
+#   elif defined(MAP_HUGE_SHIFT)
+    constexpr int flag_2mb = (21 << MAP_HUGE_SHIFT);
+#   else
+    constexpr int flag_2mb = 0;
+#   endif
+
+    void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE | flag_2mb, 0, 0);
+
 #   endif
 
     return mem == MAP_FAILED ? nullptr : mem;

src/crypto/rx/RxDataset.cpp

@@ -193,6 +193,12 @@ void xmrig::RxDataset::allocate(bool hugePages, bool oneGbPages)
     }
 
     m_memory  = new VirtualMemory(maxSize(), hugePages, oneGbPages, false, m_node);
+
+    if (m_memory->isOneGbPages()) {
+        m_scratchpadOffset = maxSize() + RANDOMX_CACHE_MAX_SIZE;
+        m_scratchpadLimit = m_memory->capacity();
+    }
+
     m_dataset = randomx_create_dataset(m_memory->raw());
 
 #   ifdef XMRIG_OS_LINUX
@@ -201,3 +207,19 @@ void xmrig::RxDataset::allocate(bool hugePages, bool oneGbPages)
     }
 #   endif
 }
+
+
+uint8_t* xmrig::RxDataset::tryAllocateScrathpad()
+{
+    uint8_t* p = reinterpret_cast<uint8_t*>(raw());
+    if (!p) {
+        return nullptr;
+    }
+
+    const size_t offset = m_scratchpadOffset.fetch_add(RANDOMX_SCRATCHPAD_L3_MAX_SIZE);
+    if (offset + RANDOMX_SCRATCHPAD_L3_MAX_SIZE > m_scratchpadLimit) {
+        return nullptr;
+    }
+
+    return p + offset;
+}

src/crypto/rx/RxDataset.h

@@ -36,6 +36,8 @@
 #include "crypto/randomx/randomx.h"
 #include "crypto/rx/RxConfig.h"
 
+#include <atomic>
+
 
 struct randomx_dataset;
 
@@ -69,6 +71,8 @@ public:
     void *raw() const;
     void setRaw(const void *raw);
 
+    uint8_t *tryAllocateScrathpad();
+
     static inline constexpr size_t maxSize() { return RANDOMX_DATASET_MAX_SIZE; }
 
 private:
@@ -79,6 +83,9 @@ private:
     randomx_dataset *m_dataset  = nullptr;
     RxCache *m_cache            = nullptr;
     VirtualMemory *m_memory     = nullptr;
+
+    std::atomic<size_t> m_scratchpadOffset;
+    size_t m_scratchpadLimit    = 0;
 };