Fixed memory allocation checks.

src/backend/common/interfaces/IRxStorage.h

@@ -45,6 +45,7 @@ class IRxStorage
 public:
     virtual ~IRxStorage() = default;
 
+    virtual bool isAllocated() const                                                                                            = 0;
     virtual HugePagesInfo hugePages() const                                                                                     = 0;
     virtual RxDataset *dataset(const Job &job, uint32_t nodeId) const                                                           = 0;
     virtual void init(const RxSeed &seed, uint32_t threads, bool hugePages, bool oneGbPages, RxConfig::Mode mode, int priority) = 0;

src/crypto/randomx/randomx.cpp

@@ -285,6 +285,10 @@ RandomX_ConfigurationBase RandomX_CurrentConfig;
 extern "C" {
 
 	randomx_cache *randomx_create_cache(randomx_flags flags, uint8_t *memory) {
+		if (!memory) {
+			return nullptr;
+		}
+
 		randomx_cache *cache = nullptr;
 
 		try {
@@ -329,6 +333,10 @@ extern "C" {
 	}
 
 	randomx_dataset *randomx_create_dataset(uint8_t *memory) {
+		if (!memory) {
+			return nullptr;
+		}
+
 		auto dataset = new randomx_dataset();
 		dataset->memory = memory;
 

src/crypto/rx/RxBasicStorage.cpp

@@ -48,13 +48,11 @@ public:
     XMRIG_DISABLE_COPY_MOVE(RxBasicStoragePrivate)
 
     inline RxBasicStoragePrivate() = default;
-    inline ~RxBasicStoragePrivate()
-    {
-        delete m_dataset;
-    }
+    inline ~RxBasicStoragePrivate() { deleteDataset(); }
 
     inline bool isReady(const Job &job) const   { return m_ready && m_seed == job; }
     inline RxDataset *dataset() const           { return m_dataset; }
+    inline void deleteDataset()                 { delete m_dataset; m_dataset = nullptr; }
 
 
     inline void setSeed(const RxSeed &seed)
@@ -69,12 +67,22 @@ public:
     }
 
 
-    inline void createDataset(bool hugePages, bool oneGbPages, RxConfig::Mode mode)
+    inline bool createDataset(bool hugePages, bool oneGbPages, RxConfig::Mode mode)
     {
         const uint64_t ts = Chrono::steadyMSecs();
 
         m_dataset = new RxDataset(hugePages, oneGbPages, true, mode, 0);
+        if (!m_dataset->cache()->get()) {
+            deleteDataset();
+
+            LOG_INFO("%s" RED_BOLD("failed to allocate RandomX memory") BLACK_BOLD(" (%" PRIu64 " ms)"), rx_tag(), Chrono::steadyMSecs() - ts);
+
+            return false;
+        }
+
         printAllocStatus(ts);
+
+        return true;
     }
 
 
@@ -82,11 +90,11 @@ public:
     {
         const uint64_t ts = Chrono::steadyMSecs();
 
-        m_dataset->init(m_seed.data(), threads, priority);
+        m_ready = m_dataset->init(m_seed.data(), threads, priority);
 
+        if (m_ready) {
             LOG_INFO("%s" GREEN_BOLD("dataset ready") BLACK_BOLD(" (%" PRIu64 " ms)"), rx_tag(), Chrono::steadyMSecs() - ts);
-
-        m_ready = true;
+        }
     }
 
 
@@ -136,6 +144,12 @@ xmrig::RxBasicStorage::~RxBasicStorage()
 }
 
 
+bool xmrig::RxBasicStorage::isAllocated() const
+{
+    return d_ptr->dataset() && d_ptr->dataset()->cache() && d_ptr->dataset()->cache()->get();
+}
+
+
 xmrig::HugePagesInfo xmrig::RxBasicStorage::hugePages() const
 {
     if (!d_ptr->dataset()) {
@@ -160,8 +174,8 @@ void xmrig::RxBasicStorage::init(const RxSeed &seed, uint32_t threads, bool huge
 {
     d_ptr->setSeed(seed);
 
-    if (!d_ptr->dataset()) {
-        d_ptr->createDataset(hugePages, oneGbPages, mode);
+    if (!d_ptr->dataset() && !d_ptr->createDataset(hugePages, oneGbPages, mode)) {
+        return;
     }
 
     d_ptr->initDataset(threads, priority);

src/crypto/rx/RxBasicStorage.h

@@ -48,6 +48,7 @@ public:
     ~RxBasicStorage() override;
 
 protected:
+    bool isAllocated() const override;
     HugePagesInfo hugePages() const override;
     RxDataset *dataset(const Job &job, uint32_t nodeId) const override;
     void init(const RxSeed &seed, uint32_t threads, bool hugePages, bool oneGbPages, RxConfig::Mode mode, int priority) override;

src/crypto/rx/RxCache.cpp

@@ -31,8 +31,6 @@
 
 
 static_assert(RANDOMX_FLAG_JIT == 8, "RANDOMX_FLAG_JIT flag mismatch");
-static_assert(RANDOMX_FLAG_LARGE_PAGES == 1, "RANDOMX_FLAG_LARGE_PAGES flag mismatch");
-
 
 
 xmrig::RxCache::RxCache(bool hugePages, uint32_t nodeId)
@@ -64,9 +62,14 @@ bool xmrig::RxCache::init(const Buffer &seed)
     }
 
     m_seed = seed;
+
+    if (m_cache) {
         randomx_init_cache(m_cache, m_seed.data(), m_seed.size());
 
         return true;
+    }
+
+    return false;
 }
 
 
@@ -78,6 +81,10 @@ xmrig::HugePagesInfo xmrig::RxCache::hugePages() const
 
 void xmrig::RxCache::create(uint8_t *memory)
 {
+    if (!memory) {
+        return;
+    }
+
     m_cache = randomx_create_cache(RANDOMX_FLAG_JIT, memory);
 
     if (!m_cache) {

src/crypto/rx/RxDataset.cpp

@@ -88,7 +88,7 @@ xmrig::RxDataset::~RxDataset()
 
 bool xmrig::RxDataset::init(const Buffer &seed, uint32_t numThreads, int priority)
 {
-    if (!m_cache) {
+    if (!m_cache || !m_cache->get()) {
         return false;
     }
 

src/crypto/rx/RxNUMAStorage.cpp

@@ -120,7 +120,7 @@ public:
     }
 
 
-    inline void createDatasets(bool hugePages, bool oneGbPages)
+    inline bool createDatasets(bool hugePages, bool oneGbPages)
     {
         const uint64_t ts = Chrono::steadyMSecs();
 
@@ -133,6 +133,10 @@ public:
         if (isCacheRequired()) {
             std::thread thread(allocateCache, this, m_nodeset.front(), hugePages);
             thread.join();
+
+            if (!m_cache) {
+                return false;
+            }
         }
 
         if (m_datasets.empty()) {
@@ -149,6 +153,8 @@ public:
         }
 
         m_allocated = true;
+
+        return true;
     }
 
 
@@ -237,6 +243,13 @@ private:
         bindToNUMANode(nodeId);
 
         auto cache = new RxCache(hugePages, nodeId);
+        if (!cache->get()) {
+            delete cache;
+
+            LOG_INFO("%s" RED_BOLD("failed to allocate RandomX memory") BLACK_BOLD(" (%" PRIu64 " ms)"), rx_tag(), Chrono::steadyMSecs() - ts);
+
+            return;
+        }
 
         std::lock_guard<std::mutex> lock(mutex);
         d_ptr->m_cache = cache;
@@ -336,6 +349,12 @@ xmrig::RxNUMAStorage::~RxNUMAStorage()
 }
 
 
+bool xmrig::RxNUMAStorage::isAllocated() const
+{
+    return d_ptr->isAllocated();
+}
+
+
 xmrig::HugePagesInfo xmrig::RxNUMAStorage::hugePages() const
 {
     if (!d_ptr->isAllocated()) {
@@ -360,8 +379,8 @@ void xmrig::RxNUMAStorage::init(const RxSeed &seed, uint32_t threads, bool hugeP
 {
     d_ptr->setSeed(seed);
 
-    if (!d_ptr->isAllocated()) {
-        d_ptr->createDatasets(hugePages, oneGbPages);
+    if (!d_ptr->isAllocated() && !d_ptr->createDatasets(hugePages, oneGbPages)) {
+        return;
     }
 
     d_ptr->initDatasets(threads, priority);

src/crypto/rx/RxNUMAStorage.h

@@ -51,6 +51,7 @@ public:
     ~RxNUMAStorage() override;
 
 protected:
+    bool isAllocated() const override;
     HugePagesInfo hugePages() const override;
     RxDataset *dataset(const Job &job, uint32_t nodeId) const override;
     void init(const RxSeed &seed, uint32_t threads, bool hugePages, bool oneGbPages, RxConfig::Mode mode, int priority) override;

src/crypto/rx/RxQueue.cpp

@@ -126,7 +126,7 @@ void xmrig::RxQueue::enqueue(const RxSeed &seed, const std::vector<uint32_t> &no
 
 bool xmrig::RxQueue::isReadyUnsafe(const Job &job) const
 {
-    return m_storage != nullptr && m_state == STATE_IDLE && m_seed == job;
+    return m_storage != nullptr && m_storage->isAllocated() && m_state == STATE_IDLE && m_seed == job;
 }