Added support for flexible huge page sizes on Linux.

This commit is contained in:
XMRig 2021-01-31 23:38:57 +07:00
parent 8faef28e7d
commit 09624c4f9b
No known key found for this signature in database
GPG key ID: 446A53638BE94409
11 changed files with 127 additions and 137 deletions

View file

@ -1,12 +1,6 @@
/* XMRig /* XMRig
* Copyright 2010 Jeff Garzik <jgarzik@pobox.com> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright 2012-2014 pooler <pooler@litecoinpool.org> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* Copyright 2014 Lucas Jones <https://github.com/lucasjones>
* Copyright 2014-2016 Wolf9466 <https://github.com/OhGodAPet>
* Copyright 2016 Jay D Dee <jayddee246@gmail.com>
* Copyright 2017-2018 XMR-Stak <https://github.com/fireice-uk>, <https://github.com/psychocrypt>
* Copyright 2018-2020 SChernykh <https://github.com/SChernykh>
* Copyright 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -77,7 +71,7 @@ rapidjson::Value xmrig::CpuConfig::toJSON(rapidjson::Document &doc) const
Value obj(kObjectType); Value obj(kObjectType);
obj.AddMember(StringRef(kEnabled), m_enabled, allocator); obj.AddMember(StringRef(kEnabled), m_enabled, allocator);
obj.AddMember(StringRef(kHugePages), m_hugePages, allocator); obj.AddMember(StringRef(kHugePages), m_hugePageSize == 0 || m_hugePageSize == kDefaultHugePageSizeKb ? Value(isHugePages()) : Value(m_hugePageSize), allocator);
obj.AddMember(StringRef(kHugePagesJit), m_hugePagesJit, allocator); obj.AddMember(StringRef(kHugePagesJit), m_hugePagesJit, allocator);
obj.AddMember(StringRef(kHwAes), m_aes == AES_AUTO ? Value(kNullType) : Value(m_aes == AES_HW), allocator); obj.AddMember(StringRef(kHwAes), m_aes == AES_AUTO ? Value(kNullType) : Value(m_aes == AES_HW), allocator);
obj.AddMember(StringRef(kPriority), priority() != -1 ? Value(priority()) : Value(kNullType), allocator); obj.AddMember(StringRef(kPriority), priority() != -1 ? Value(priority()) : Value(kNullType), allocator);
@ -137,14 +131,14 @@ void xmrig::CpuConfig::read(const rapidjson::Value &value)
{ {
if (value.IsObject()) { if (value.IsObject()) {
m_enabled = Json::getBool(value, kEnabled, m_enabled); m_enabled = Json::getBool(value, kEnabled, m_enabled);
m_hugePages = Json::getBool(value, kHugePages, m_hugePages);
m_hugePagesJit = Json::getBool(value, kHugePagesJit, m_hugePagesJit); m_hugePagesJit = Json::getBool(value, kHugePagesJit, m_hugePagesJit);
m_limit = Json::getUint(value, kMaxThreadsHint, m_limit); m_limit = Json::getUint(value, kMaxThreadsHint, m_limit);
m_yield = Json::getBool(value, kYield, m_yield); m_yield = Json::getBool(value, kYield, m_yield);
setAesMode(Json::getValue(value, kHwAes)); setAesMode(Json::getValue(value, kHwAes));
setPriority(Json::getInt(value, kPriority, -1)); setHugePages(Json::getValue(value, kHugePages));
setMemoryPool(Json::getValue(value, kMemoryPool)); setMemoryPool(Json::getValue(value, kMemoryPool));
setPriority(Json::getInt(value, kPriority, -1));
# ifdef XMRIG_FEATURE_ASM # ifdef XMRIG_FEATURE_ASM
m_assembly = Json::getValue(value, kAsm); m_assembly = Json::getValue(value, kAsm);
@ -218,6 +212,19 @@ void xmrig::CpuConfig::setAesMode(const rapidjson::Value &value)
} }
void xmrig::CpuConfig::setHugePages(const rapidjson::Value &value)
{
if (value.IsBool()) {
m_hugePageSize = value.GetBool() ? kDefaultHugePageSizeKb : 0U;
}
else if (value.IsUint()) {
const uint32_t size = value.GetUint();
m_hugePageSize = size < kOneGbPageSizeKb ? size : kDefaultHugePageSizeKb;
}
}
void xmrig::CpuConfig::setMemoryPool(const rapidjson::Value &value) void xmrig::CpuConfig::setMemoryPool(const rapidjson::Value &value)
{ {
if (value.IsBool()) { if (value.IsBool()) {

View file

@ -1,12 +1,6 @@
/* XMRig /* XMRig
* Copyright 2010 Jeff Garzik <jgarzik@pobox.com> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright 2012-2014 pooler <pooler@litecoinpool.org> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* Copyright 2014 Lucas Jones <https://github.com/lucasjones>
* Copyright 2014-2016 Wolf9466 <https://github.com/OhGodAPet>
* Copyright 2016 Jay D Dee <jayddee246@gmail.com>
* Copyright 2017-2018 XMR-Stak <https://github.com/fireice-uk>, <https://github.com/psychocrypt>
* Copyright 2018-2020 SChernykh <https://github.com/SChernykh>
* Copyright 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -75,8 +69,9 @@ public:
std::vector<CpuLaunchData> get(const Miner *miner, const Algorithm &algorithm) const; std::vector<CpuLaunchData> get(const Miner *miner, const Algorithm &algorithm) const;
void read(const rapidjson::Value &value); void read(const rapidjson::Value &value);
inline bool astrobwtAVX2() const { return m_astrobwtAVX2; }
inline bool isEnabled() const { return m_enabled; } inline bool isEnabled() const { return m_enabled; }
inline bool isHugePages() const { return m_hugePages; } inline bool isHugePages() const { return m_hugePageSize > 0; }
inline bool isHugePagesJit() const { return m_hugePagesJit; } inline bool isHugePagesJit() const { return m_hugePagesJit; }
inline bool isShouldSave() const { return m_shouldSave; } inline bool isShouldSave() const { return m_shouldSave; }
inline bool isYield() const { return m_yield; } inline bool isYield() const { return m_yield; }
@ -84,13 +79,17 @@ public:
inline const String &argon2Impl() const { return m_argon2Impl; } inline const String &argon2Impl() const { return m_argon2Impl; }
inline const Threads<CpuThreads> &threads() const { return m_threads; } inline const Threads<CpuThreads> &threads() const { return m_threads; }
inline int astrobwtMaxSize() const { return m_astrobwtMaxSize; } inline int astrobwtMaxSize() const { return m_astrobwtMaxSize; }
inline bool astrobwtAVX2() const { return m_astrobwtAVX2; }
inline int priority() const { return m_priority; } inline int priority() const { return m_priority; }
inline size_t hugePageSize() const { return m_hugePageSize * 1024U; }
inline uint32_t limit() const { return m_limit; } inline uint32_t limit() const { return m_limit; }
private: private:
constexpr static size_t kDefaultHugePageSizeKb = 2048U;
constexpr static size_t kOneGbPageSizeKb = 1048576U;
void generate(); void generate();
void setAesMode(const rapidjson::Value &value); void setAesMode(const rapidjson::Value &value);
void setHugePages(const rapidjson::Value &value);
void setMemoryPool(const rapidjson::Value &value); void setMemoryPool(const rapidjson::Value &value);
inline void setPriority(int priority) { m_priority = (priority >= -1 && priority <= 5) ? priority : -1; } inline void setPriority(int priority) { m_priority = (priority >= -1 && priority <= 5) ? priority : -1; }
@ -99,13 +98,13 @@ private:
Assembly m_assembly; Assembly m_assembly;
bool m_astrobwtAVX2 = false; bool m_astrobwtAVX2 = false;
bool m_enabled = true; bool m_enabled = true;
bool m_hugePages = true;
bool m_hugePagesJit = false; bool m_hugePagesJit = false;
bool m_shouldSave = false; bool m_shouldSave = false;
bool m_yield = true; bool m_yield = true;
int m_astrobwtMaxSize = 550; int m_astrobwtMaxSize = 550;
int m_memoryPool = 0; int m_memoryPool = 0;
int m_priority = -1; int m_priority = -1;
size_t m_hugePageSize = kDefaultHugePageSizeKb;
String m_argon2Impl; String m_argon2Impl;
Threads<CpuThreads> m_threads; Threads<CpuThreads> m_threads;
uint32_t m_limit = 100; uint32_t m_limit = 100;

View file

@ -50,7 +50,7 @@ int xmrig::Controller::init()
{ {
Base::init(); Base::init();
VirtualMemory::init(config()->cpu().memPoolSize(), config()->cpu().isHugePages()); VirtualMemory::init(config()->cpu().memPoolSize(), config()->cpu().hugePageSize());
m_network = std::make_shared<Network>(this); m_network = std::make_shared<Network>(this);

View file

@ -1,12 +1,6 @@
/* XMRig /* XMRig
* Copyright 2010 Jeff Garzik <jgarzik@pobox.com> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright 2012-2014 pooler <pooler@litecoinpool.org> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* Copyright 2014 Lucas Jones <https://github.com/lucasjones>
* Copyright 2014-2016 Wolf9466 <https://github.com/OhGodAPet>
* Copyright 2016 Jay D Dee <jayddee246@gmail.com>
* Copyright 2017-2018 XMR-Stak <https://github.com/fireice-uk>, <https://github.com/psychocrypt>
* Copyright 2018-2019 SChernykh <https://github.com/SChernykh>
* Copyright 2016-2019 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -27,24 +21,16 @@
#include "crypto/common/VirtualMemory.h" #include "crypto/common/VirtualMemory.h"
namespace xmrig {
constexpr size_t twoMiB = 2U * 1024U * 1024U;
constexpr size_t oneGiB = 1024U * 1024U * 1024U;
} // namespace xmrig
xmrig::HugePagesInfo::HugePagesInfo(const VirtualMemory *memory) xmrig::HugePagesInfo::HugePagesInfo(const VirtualMemory *memory)
{ {
if (memory->isOneGbPages()) { if (memory->isOneGbPages()) {
size = VirtualMemory::align(memory->size(), oneGiB); size = VirtualMemory::align(memory->size(), VirtualMemory::kOneGiB);
total = size / oneGiB; total = size / VirtualMemory::kOneGiB;
allocated = size / oneGiB; allocated = size / VirtualMemory::kOneGiB;
} }
else { else {
size = memory->size(); size = VirtualMemory::alignToHugePageSize(memory->size());
total = size / twoMiB; total = size / VirtualMemory::hugePageSize();
allocated = memory->isHugePages() ? total : 0; allocated = memory->isHugePages() ? total : 0;
} }
} }

View file

@ -1,12 +1,6 @@
/* XMRig /* XMRig
* Copyright 2010 Jeff Garzik <jgarzik@pobox.com> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright 2012-2014 pooler <pooler@litecoinpool.org> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* Copyright 2014 Lucas Jones <https://github.com/lucasjones>
* Copyright 2014-2016 Wolf9466 <https://github.com/OhGodAPet>
* Copyright 2016 Jay D Dee <jayddee246@gmail.com>
* Copyright 2017-2018 XMR-Stak <https://github.com/fireice-uk>, <https://github.com/psychocrypt>
* Copyright 2018-2019 SChernykh <https://github.com/SChernykh>
* Copyright 2016-2019 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by

View file

@ -1,6 +1,6 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -18,8 +18,6 @@
#include "crypto/common/LinuxMemory.h" #include "crypto/common/LinuxMemory.h"
#include "3rdparty/fmt/core.h" #include "3rdparty/fmt/core.h"
#include "backend/cpu/Cpu.h"
#include "base/io/log/Log.h"
#include "crypto/common/VirtualMemory.h" #include "crypto/common/VirtualMemory.h"
@ -37,33 +35,32 @@ constexpr size_t twoMiB = 2U * 1024U * 1024U;
constexpr size_t oneGiB = 1024U * 1024U * 1024U; constexpr size_t oneGiB = 1024U * 1024U * 1024U;
static inline std::string sysfs_path(uint32_t node, bool oneGbPages, bool nr) static inline std::string sysfs_path(uint32_t node, size_t hugePageSize, bool nr)
{ {
return fmt::format("/sys/devices/system/node/node{}/hugepages/hugepages-{}kB/{}_hugepages", node, oneGbPages ? "1048576" : "2048", nr ? "nr" : "free"); return fmt::format("/sys/devices/system/node/node{}/hugepages/hugepages-{}kB/{}_hugepages", node, hugePageSize / 1024, nr ? "nr" : "free");
} }
static inline bool write_nr_hugepages(uint32_t node, bool oneGbPages, uint64_t count) { return LinuxMemory::write(sysfs_path(node, oneGbPages, true).c_str(), count); } static inline bool write_nr_hugepages(uint32_t node, size_t hugePageSize, uint64_t count) { return LinuxMemory::write(sysfs_path(node, hugePageSize, true).c_str(), count); }
static inline int64_t free_hugepages(uint32_t node, bool oneGbPages) { return LinuxMemory::read(sysfs_path(node, oneGbPages, false).c_str()); } static inline int64_t free_hugepages(uint32_t node, size_t hugePageSize) { return LinuxMemory::read(sysfs_path(node, hugePageSize, false).c_str()); }
static inline int64_t nr_hugepages(uint32_t node, bool oneGbPages) { return LinuxMemory::read(sysfs_path(node, oneGbPages, true).c_str()); } static inline int64_t nr_hugepages(uint32_t node, size_t hugePageSize) { return LinuxMemory::read(sysfs_path(node, hugePageSize, true).c_str()); }
} // namespace xmrig } // namespace xmrig
bool xmrig::LinuxMemory::reserve(size_t size, uint32_t node, bool oneGbPages) bool xmrig::LinuxMemory::reserve(size_t size, uint32_t node, size_t hugePageSize)
{ {
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
const size_t pageSize = oneGbPages ? oneGiB : twoMiB; const size_t required = VirtualMemory::align(size, hugePageSize) / hugePageSize;
const size_t required = VirtualMemory::align(size, pageSize) / pageSize;
const auto available = free_hugepages(node, oneGbPages); const auto available = free_hugepages(node, hugePageSize);
if (available < 0 || static_cast<size_t>(available) >= required) { if (available < 0 || static_cast<size_t>(available) >= required) {
return false; return false;
} }
return write_nr_hugepages(node, oneGbPages, std::max<size_t>(nr_hugepages(node, oneGbPages), 0) + (required - available)); return write_nr_hugepages(node, hugePageSize, std::max<size_t>(nr_hugepages(node, hugePageSize), 0) + (required - available));
} }

View file

@ -1,6 +1,6 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -30,7 +30,7 @@ namespace xmrig {
class LinuxMemory class LinuxMemory
{ {
public: public:
static bool reserve(size_t size, uint32_t node, bool oneGbPages = false); static bool reserve(size_t size, uint32_t node, size_t hugePageSize);
static bool write(const char *path, uint64_t value); static bool write(const char *path, uint64_t value);
static int64_t read(const char *path); static int64_t read(const char *path);

View file

@ -1,7 +1,7 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 tevador <tevador@gmail.com> * Copyright (c) 2018-2020 tevador <tevador@gmail.com>
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -36,16 +36,19 @@
namespace xmrig { namespace xmrig {
static IMemoryPool *pool = nullptr;
size_t VirtualMemory::m_hugePageSize = VirtualMemory::kDefaultHugePageSize;
static IMemoryPool *pool = nullptr;
static std::mutex mutex; static std::mutex mutex;
} // namespace xmrig } // namespace xmrig
xmrig::VirtualMemory::VirtualMemory(size_t size, bool hugePages, bool oneGbPages, bool usePool, uint32_t node, size_t alignSize) : xmrig::VirtualMemory::VirtualMemory(size_t size, bool hugePages, bool oneGbPages, bool usePool, uint32_t node, size_t alignSize) :
m_size(align(size)), m_size(alignToHugePageSize(size)),
m_capacity(m_size), m_node(node),
m_node(node) m_capacity(m_size)
{ {
if (usePool) { if (usePool) {
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
@ -114,18 +117,18 @@ void xmrig::VirtualMemory::destroy()
} }
void xmrig::VirtualMemory::init(size_t poolSize, bool hugePages) void xmrig::VirtualMemory::init(size_t poolSize, size_t hugePageSize)
{ {
if (!pool) { if (!pool) {
osInit(hugePages); osInit(hugePageSize);
} }
# ifdef XMRIG_FEATURE_HWLOC # ifdef XMRIG_FEATURE_HWLOC
if (Cpu::info()->nodes() > 1) { if (Cpu::info()->nodes() > 1) {
pool = new NUMAMemoryPool(align(poolSize, Cpu::info()->nodes()), hugePages); pool = new NUMAMemoryPool(align(poolSize, Cpu::info()->nodes()), hugePageSize > 0);
} else } else
# endif # endif
{ {
pool = new MemoryPool(poolSize, hugePages); pool = new MemoryPool(poolSize, hugePageSize > 0);
} }
} }

View file

@ -1,7 +1,7 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 tevador <tevador@gmail.com> * Copyright (c) 2018-2020 tevador <tevador@gmail.com>
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -39,6 +39,9 @@ class VirtualMemory
public: public:
XMRIG_DISABLE_COPY_MOVE_DEFAULT(VirtualMemory) XMRIG_DISABLE_COPY_MOVE_DEFAULT(VirtualMemory)
constexpr static size_t kDefaultHugePageSize = 2U * 1024U * 1024U;
constexpr static size_t kOneGiB = 1024U * 1024U * 1024U;
VirtualMemory(size_t size, bool hugePages, bool oneGbPages, bool usePool, uint32_t node = 0, size_t alignSize = 64); VirtualMemory(size_t size, bool hugePages, bool oneGbPages, bool usePool, uint32_t node = 0, size_t alignSize = 64);
~VirtualMemory(); ~VirtualMemory();
@ -65,9 +68,11 @@ public:
static void destroy(); static void destroy();
static void flushInstructionCache(void *p, size_t size); static void flushInstructionCache(void *p, size_t size);
static void freeLargePagesMemory(void *p, size_t size); static void freeLargePagesMemory(void *p, size_t size);
static void init(size_t poolSize, bool hugePages); static void init(size_t poolSize, size_t hugePageSize);
static inline constexpr size_t align(size_t pos, size_t align = 2097152) { return ((pos - 1) / align + 1) * align; } static inline constexpr size_t align(size_t pos, size_t align = kDefaultHugePageSize) { return ((pos - 1) / align + 1) * align; }
static inline size_t alignToHugePageSize(size_t pos) { return align(pos, hugePageSize()); }
static inline size_t hugePageSize() { return m_hugePageSize; }
private: private:
enum Flags { enum Flags {
@ -78,15 +83,17 @@ private:
FLAG_MAX FLAG_MAX
}; };
static void osInit(bool hugePages); static void osInit(size_t hugePageSize);
bool allocateLargePagesMemory(); bool allocateLargePagesMemory();
bool allocateOneGbPagesMemory(); bool allocateOneGbPagesMemory();
void freeLargePagesMemory(); void freeLargePagesMemory();
static size_t m_hugePageSize;
const size_t m_size; const size_t m_size;
size_t m_capacity;
const uint32_t m_node; const uint32_t m_node;
size_t m_capacity;
std::bitset<FLAG_MAX> m_flags; std::bitset<FLAG_MAX> m_flags;
uint8_t *m_scratchpad = nullptr; uint8_t *m_scratchpad = nullptr;
}; };

View file

@ -1,7 +1,7 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 tevador <tevador@gmail.com> * Copyright (c) 2018-2020 tevador <tevador@gmail.com>
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -18,13 +18,14 @@
*/ */
#include <cstdlib> #include "crypto/common/VirtualMemory.h"
#include <sys/mman.h>
#include "backend/cpu/Cpu.h" #include "backend/cpu/Cpu.h"
#include "crypto/common/portable/mm_malloc.h" #include "crypto/common/portable/mm_malloc.h"
#include "crypto/common/VirtualMemory.h"
#include <cmath>
#include <cstdlib>
#include <sys/mman.h>
#ifdef XMRIG_OS_APPLE #ifdef XMRIG_OS_APPLE
@ -42,14 +43,21 @@
#endif #endif
#if defined(XMRIG_OS_LINUX) #ifdef XMRIG_OS_LINUX
# if (defined(MAP_HUGE_1GB) || defined(MAP_HUGE_SHIFT))
# define XMRIG_HAS_1GB_PAGES
# endif
# include "crypto/common/LinuxMemory.h" # include "crypto/common/LinuxMemory.h"
#endif #endif
#ifndef MAP_HUGE_SHIFT
# define MAP_HUGE_SHIFT 26
#endif
#ifndef MAP_HUGE_MASK
# define MAP_HUGE_MASK 0x3f
#endif
#ifdef XMRIG_SECURE_JIT #ifdef XMRIG_SECURE_JIT
# define SECURE_PROT_EXEC 0 # define SECURE_PROT_EXEC 0
#else #else
@ -57,6 +65,18 @@
#endif #endif
namespace xmrig {
static inline int hugePagesFlag(size_t size)
{
return (static_cast<int>(log2(size)) & MAP_HUGE_MASK) << MAP_HUGE_SHIFT;
}
} // namespace xmrig
bool xmrig::VirtualMemory::isHugepagesAvailable() bool xmrig::VirtualMemory::isHugepagesAvailable()
{ {
# if defined(XMRIG_OS_MACOS) && defined(XMRIG_ARM) # if defined(XMRIG_OS_MACOS) && defined(XMRIG_ARM)
@ -69,7 +89,7 @@ bool xmrig::VirtualMemory::isHugepagesAvailable()
bool xmrig::VirtualMemory::isOneGbPagesAvailable() bool xmrig::VirtualMemory::isOneGbPagesAvailable()
{ {
# ifdef XMRIG_HAS_1GB_PAGES # ifdef XMRIG_OS_LINUX
return Cpu::info()->hasOneGbPages(); return Cpu::info()->hasOneGbPages();
# else # else
return false; return false;
@ -126,18 +146,10 @@ void *xmrig::VirtualMemory::allocateExecutableMemory(size_t size, bool hugePages
# else # else
# if defined(MAP_HUGE_2MB)
constexpr int flag_2mb = MAP_HUGE_2MB;
# elif defined(MAP_HUGE_SHIFT)
constexpr int flag_2mb = (21 << MAP_HUGE_SHIFT);
# else
constexpr int flag_2mb = 0;
# endif
void *mem = nullptr; void *mem = nullptr;
if (hugePages) { if (hugePages) {
mem = mmap(0, align(size), PROT_READ | PROT_WRITE | SECURE_PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE | flag_2mb, -1, 0); mem = mmap(0, align(size), PROT_READ | PROT_WRITE | SECURE_PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE | hugePagesFlag(hugePageSize()), -1, 0);
} }
if (!mem) { if (!mem) {
@ -157,17 +169,7 @@ void *xmrig::VirtualMemory::allocateLargePagesMemory(size_t size)
# elif defined(__FreeBSD__) # elif defined(__FreeBSD__)
void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER | MAP_PREFAULT_READ, -1, 0); void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER | MAP_PREFAULT_READ, -1, 0);
# else # else
void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE | hugePagesFlag(hugePageSize()), 0, 0);
# if defined(MAP_HUGE_2MB)
constexpr int flag_2mb = MAP_HUGE_2MB;
# elif defined(MAP_HUGE_SHIFT)
constexpr int flag_2mb = (21 << MAP_HUGE_SHIFT);
# else
constexpr int flag_2mb = 0;
# endif
void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE | flag_2mb, 0, 0);
# endif # endif
return mem == MAP_FAILED ? nullptr : mem; return mem == MAP_FAILED ? nullptr : mem;
@ -176,17 +178,9 @@ void *xmrig::VirtualMemory::allocateLargePagesMemory(size_t size)
void *xmrig::VirtualMemory::allocateOneGbPagesMemory(size_t size) void *xmrig::VirtualMemory::allocateOneGbPagesMemory(size_t size)
{ {
# ifdef XMRIG_HAS_1GB_PAGES # ifdef XMRIG_OS_LINUX
if (isOneGbPagesAvailable()) { if (isOneGbPagesAvailable()) {
# if defined(MAP_HUGE_1GB) void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE | hugePagesFlag(kOneGiB), 0, 0);
constexpr int flag_1gb = MAP_HUGE_1GB;
# elif defined(MAP_HUGE_SHIFT)
constexpr int flag_1gb = (30 << MAP_HUGE_SHIFT);
# else
constexpr int flag_1gb = 0;
# endif
void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE | flag_1gb, 0, 0);
return mem == MAP_FAILED ? nullptr : mem; return mem == MAP_FAILED ? nullptr : mem;
} }
@ -212,15 +206,18 @@ void xmrig::VirtualMemory::freeLargePagesMemory(void *p, size_t size)
} }
void xmrig::VirtualMemory::osInit(bool) void xmrig::VirtualMemory::osInit(size_t hugePageSize)
{ {
if (hugePageSize) {
m_hugePageSize = hugePageSize;
}
} }
bool xmrig::VirtualMemory::allocateLargePagesMemory() bool xmrig::VirtualMemory::allocateLargePagesMemory()
{ {
# if defined(XMRIG_OS_LINUX) # ifdef XMRIG_OS_LINUX
LinuxMemory::reserve(m_size, m_node); LinuxMemory::reserve(m_size, m_node, hugePageSize());
# endif # endif
m_scratchpad = static_cast<uint8_t*>(allocateLargePagesMemory(m_size)); m_scratchpad = static_cast<uint8_t*>(allocateLargePagesMemory(m_size));
@ -242,8 +239,8 @@ bool xmrig::VirtualMemory::allocateLargePagesMemory()
bool xmrig::VirtualMemory::allocateOneGbPagesMemory() bool xmrig::VirtualMemory::allocateOneGbPagesMemory()
{ {
# if defined(XMRIG_HAS_1GB_PAGES) # ifdef XMRIG_OS_LINUX
LinuxMemory::reserve(m_size, m_node, true); LinuxMemory::reserve(m_size, m_node, kOneGiB);
# endif # endif
m_scratchpad = static_cast<uint8_t*>(allocateOneGbPagesMemory(m_size)); m_scratchpad = static_cast<uint8_t*>(allocateOneGbPagesMemory(m_size));

View file

@ -1,7 +1,7 @@
/* XMRig /* XMRig
* Copyright (c) 2018-2020 tevador <tevador@gmail.com> * Copyright (c) 2018-2020 tevador <tevador@gmail.com>
* Copyright (c) 2018-2020 SChernykh <https://github.com/SChernykh> * Copyright (c) 2018-2021 SChernykh <https://github.com/SChernykh>
* Copyright (c) 2016-2020 XMRig <https://github.com/xmrig>, <support@xmrig.com> * Copyright (c) 2016-2021 XMRig <https://github.com/xmrig>, <support@xmrig.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -24,9 +24,9 @@
#include <tchar.h> #include <tchar.h>
#include "crypto/common/VirtualMemory.h"
#include "base/io/log/Log.h" #include "base/io/log/Log.h"
#include "crypto/common/portable/mm_malloc.h" #include "crypto/common/portable/mm_malloc.h"
#include "crypto/common/VirtualMemory.h"
#ifdef XMRIG_SECURE_JIT #ifdef XMRIG_SECURE_JIT
@ -233,9 +233,9 @@ void xmrig::VirtualMemory::freeLargePagesMemory(void *p, size_t)
} }
void xmrig::VirtualMemory::osInit(bool hugePages) void xmrig::VirtualMemory::osInit(size_t hugePageSize)
{ {
if (hugePages) { if (hugePageSize) {
hugepagesAvailable = TrySetLockPagesPrivilege(); hugepagesAvailable = TrySetLockPagesPrivilege();
} }
} }