diff --git a/src/crypto.cpp b/src/crypto.cpp index def5e30..b061b7e 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -158,6 +158,9 @@ class Cache { public: Cache() + : derivations(new DerivationsMap()) + , public_keys(new PublicKeysMap()) + , tx_keys(new TxKeysMap()) { uv_rwlock_init_checked(&derivations_lock); uv_rwlock_init_checked(&public_keys_lock); @@ -166,6 +169,10 @@ public: ~Cache() { + delete derivations; + delete public_keys; + delete tx_keys; + uv_rwlock_destroy(&derivations_lock); uv_rwlock_destroy(&public_keys_lock); uv_rwlock_destroy(&tx_keys_lock); @@ -180,8 +187,8 @@ public: derivation = {}; { ReadLock lock(derivations_lock); - auto it = derivations.find(index); - if (it != derivations.end()) { + auto it = derivations->find(index); + if (it != derivations->end()) { const DerivationEntry& entry = it->second; derivation = entry.m_derivation; if (entry.find_view_tag(output_index, view_tag)) { @@ -210,7 +217,7 @@ public: { WriteLock lock(derivations_lock); - DerivationEntry& entry = derivations.emplace(index, DerivationEntry{ derivation, {} }).first->second; + DerivationEntry& entry = derivations->emplace(index, DerivationEntry{ derivation, {} }).first->second; const uint32_t k = static_cast(output_index << 8) | view_tag; if (std::find(entry.m_viewTags.begin(), entry.m_viewTags.end(), k) == entry.m_viewTags.end()) { @@ -230,8 +237,8 @@ public: { ReadLock lock(public_keys_lock); - auto it = public_keys.find(index); - if (it != public_keys.end()) { + auto it = public_keys->find(index); + if (it != public_keys->end()) { derived_key = it->second; return true; } @@ -257,7 +264,7 @@ public: { WriteLock lock(public_keys_lock); - public_keys.emplace(index, derived_key); + public_keys->emplace(index, derived_key); } return true; @@ -271,8 +278,8 @@ public: { ReadLock lock(tx_keys_lock); - auto it = tx_keys.find(index); - if (it != tx_keys.end()) { + auto it = tx_keys->find(index); + if (it != tx_keys->end()) { pub = it->second.first; sec = it->second.second; return; @@ -291,15 +298,27 @@ public: { WriteLock lock(tx_keys_lock); - tx_keys.emplace(index, std::pair(pub, sec)); + tx_keys->emplace(index, std::pair(pub, sec)); } } void clear() { - { WriteLock lock(derivations_lock); derivations.clear(); } - { WriteLock lock(public_keys_lock); public_keys.clear(); } - { WriteLock lock(tx_keys_lock); tx_keys.clear(); } + { + WriteLock lock(derivations_lock); + delete derivations; + derivations = new DerivationsMap(); + } + { + WriteLock lock(public_keys_lock); + delete public_keys; + public_keys = new PublicKeysMap(); + } + { + WriteLock lock(tx_keys_lock); + delete tx_keys; + tx_keys = new TxKeysMap(); + } } private: @@ -319,14 +338,18 @@ private: } }; + typedef unordered_map, DerivationEntry> DerivationsMap; + typedef unordered_map, hash> PublicKeysMap; + typedef unordered_map, std::pair> TxKeysMap; + uv_rwlock_t derivations_lock; - unordered_map, DerivationEntry> derivations; + DerivationsMap* derivations; uv_rwlock_t public_keys_lock; - unordered_map, hash> public_keys; + PublicKeysMap* public_keys; uv_rwlock_t tx_keys_lock; - unordered_map, std::pair> tx_keys; + TxKeysMap* tx_keys; }; static Cache* cache = nullptr; @@ -379,7 +402,9 @@ void destroy_crypto_cache() void clear_crypto_cache() { - cache->clear(); + if (cache) { + cache->clear(); + } } } // namespace p2pool diff --git a/src/memory_leak_debug.cpp b/src/memory_leak_debug.cpp index ed0dca7..e3c4050 100644 --- a/src/memory_leak_debug.cpp +++ b/src/memory_leak_debug.cpp @@ -32,15 +32,48 @@ namespace p2pool { static bool track_memory = false; -constexpr size_t N = 2097152; -constexpr size_t MAX_FRAMES = 30; +constexpr size_t N = 1 << 22; +constexpr size_t MAX_FRAMES = 29; struct TrackedAllocation { void* p; void* stack_trace[MAX_FRAMES]; + uint64_t allocated_size; uint32_t thread_id; - uint32_t allocated_size; + + FORCEINLINE bool operator<(const TrackedAllocation& rhs) { return memcmp(stack_trace, rhs.stack_trace, sizeof(stack_trace)) < 0; } + FORCEINLINE bool operator==(const TrackedAllocation& rhs) { return memcmp(stack_trace, rhs.stack_trace, sizeof(stack_trace)) == 0; } + + void print(HANDLE h) const + { + char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)] = {}; + PSYMBOL_INFO pSymbol = reinterpret_cast(buffer); + + pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); + pSymbol->MaxNameLen = MAX_SYM_NAME; + + IMAGEHLP_LINE64 line{}; + line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + + for (size_t j = 0; j < MAX_FRAMES; ++j) { + const DWORD64 address = reinterpret_cast(stack_trace[j]); + DWORD64 t1 = 0; + DWORD t2 = 0; + if (SymFromAddr(h, address, &t1, pSymbol) && SymGetLineFromAddr64(h, address, &t2, &line)) { + const char* s = line.FileName; + const char* file_name = nullptr; + while (*s) { + if ((*s == '\\') || (*s == '/')) { + file_name = s + 1; + } + ++s; + } + printf("%-25s %s (line %lu)\n", file_name ? file_name : line.FileName, pSymbol->Name, line.LineNumber); + } + } + printf("\n"); + } }; static_assert(sizeof(TrackedAllocation) == 256, ""); @@ -51,15 +84,62 @@ uint32_t first[N]; uint32_t next[N]; TrackedAllocation allocations[N]; uint32_t num_allocations = 0; +uint64_t total_allocated = 0; uint32_t cur_allocation_index = 1; +void show_top_10() +{ + TrackedAllocation* buf = reinterpret_cast(VirtualAlloc(nullptr, sizeof(TrackedAllocation) * N, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)); + if (!buf) { + return; + } + + const HANDLE h = GetCurrentProcess(); + + { + p2pool::MutexLock lock(allocation_lock); + + TrackedAllocation* end = buf; + for (size_t i = 0; i < N; ++i) { + if (allocations[i].allocated_size) { + *(end++) = allocations[i]; + } + } + + std::sort(buf, end); + + TrackedAllocation* prev = buf; + for (TrackedAllocation* p = buf + 1; p < end; ++p) { + if (*p == *prev) { + prev->allocated_size += p->allocated_size; + } + else { + ++prev; + *prev = *p; + } + } + end = prev + 1; + + std::sort(buf, end, [](const auto& a, const auto& b) { return a.allocated_size > b.allocated_size; }); + + printf("%I64u total bytes allocated\n", total_allocated); + + for (TrackedAllocation* p = buf; (p < buf + 10) && (p < end); ++p) { + printf("%I64u bytes allocated at:\n", p->allocated_size); + p->print(h); + } + } + + VirtualFree(buf, 0, MEM_RELEASE); +} + FORCEINLINE static void add_alocation(void* p, size_t size) { if (!track_memory) { return; } - void* stack_trace[MAX_FRAMES]; + void* stack_trace[MAX_FRAMES] = {}; DWORD hash; CaptureStackBackTrace(1, MAX_FRAMES, stack_trace, &hash); @@ -74,6 +154,7 @@ FORCEINLINE static void add_alocation(void* p, size_t size) // Make N two times bigger if this triggers __debugbreak(); } + total_allocated += size; for (uint64_t i = cur_allocation_index;; i = (i + 1) & (N - 1)) { if (i && !allocations[i].allocated_size) { @@ -105,6 +186,7 @@ FORCEINLINE static void remove_allocation(void* p) for (uint32_t prev = 0, k = first[index]; k != 0; prev = k, k = next[k]) { if (allocations[k].p == p) { + total_allocated -= allocations[k].allocated_size; allocations[k].allocated_size = 0; if (prev) { next[prev] = next[k]; @@ -181,6 +263,8 @@ void* calloc_hook(size_t count, size_t size) noexcept void memory_tracking_start() { + SymInitialize(GetCurrentProcess(), NULL, TRUE); + using namespace p2pool; uv_replace_allocator(malloc_hook, realloc_hook, calloc_hook, free_hook); @@ -196,7 +280,6 @@ void memory_tracking_stop() uv_mutex_destroy(&allocation_lock); const HANDLE h = GetCurrentProcess(); - SymInitialize(h, NULL, TRUE); uint64_t total_leaks = 0; @@ -205,39 +288,16 @@ void memory_tracking_stop() if (t.allocated_size) { total_leaks += t.allocated_size; - char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)] = {}; - PSYMBOL_INFO pSymbol = reinterpret_cast(buffer); - - pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); - pSymbol->MaxNameLen = MAX_SYM_NAME; - - IMAGEHLP_LINE64 line{}; - line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); - - printf("Memory leak detected, %u bytes allocated at %p by thread %u:\n", t.allocated_size, t.p, t.thread_id); - for (size_t j = 0; j < MAX_FRAMES; ++j) { - const DWORD64 address = reinterpret_cast(t.stack_trace[j]); - DWORD64 t1 = 0; - DWORD t2 = 0; - if (SymFromAddr(h, address, &t1, pSymbol) && SymGetLineFromAddr64(h, address, &t2, &line)) { - const char* s = line.FileName; - const char* file_name = nullptr; - while (*s) { - if ((*s == '\\') || (*s == '/')) { - file_name = s + 1; - } - ++s; - } - printf("%-25s %s (line %lu)\n", file_name ? file_name : line.FileName, pSymbol->Name, line.LineNumber); - } - } - printf("\n"); + printf("Memory leak detected, %I64u bytes allocated at %p by thread %u:\n", t.allocated_size, t.p, t.thread_id); + t.print(h); } } if (total_leaks > 0) { printf("%I64u bytes leaked\n\n", total_leaks); } + + SymCleanup(h); } NOINLINE void* operator new(size_t n) { return p2pool::allocate(n); } diff --git a/src/p2pool.cpp b/src/p2pool.cpp index ae8726c..247497b 100644 --- a/src/p2pool.cpp +++ b/src/p2pool.cpp @@ -302,7 +302,9 @@ void p2pool::handle_miner_data(MinerData& data) ); // Tx secret keys from all miners change every block, so cache can be cleared here - clear_crypto_cache(); + if (m_sideChain->precalcFinished()) { + clear_crypto_cache(); + } if (!is_main_thread()) { update_block_template_async(); diff --git a/src/side_chain.cpp b/src/side_chain.cpp index 7d51e2e..e9220db 100644 --- a/src/side_chain.cpp +++ b/src/side_chain.cpp @@ -170,7 +170,10 @@ SideChain::SideChain(p2pool* pool, NetworkType type, const char* pool_name) // Use between 1 and 8 threads if (numThreads < 1) numThreads = 1; + +#ifndef _DEBUG if (numThreads > 8) numThreads = 8; +#endif LOGINFO(4, "running " << numThreads << " pre-calculation workers"); @@ -2128,6 +2131,9 @@ void SideChain::finish_precalc() LOGERR(1, "exception in finish_precalc(): " << e.what()); } + // Also clear cache because it has data from all old blocks now + clear_crypto_cache(); + #ifdef DEV_TEST_SYNC if (m_pool) { LOGINFO(0, log::LightGreen() << "[DEV] Synchronization finished successfully, stopping P2Pool now");