Merge pull request #2303

5a283078 cryptonote_protocol: large block sync size before v4 (moneromooo-monero)
7b747607 cryptonote_protocol: kick idle synchronizing peers (moneromooo-monero)

src/common/command_line.cpp

@@ -129,8 +129,8 @@ namespace command_line
   };
   const command_line::arg_descriptor<size_t> arg_block_sync_size  = {
     "block-sync-size"
-  , "How many blocks to sync at once during chain synchronization."
-  , BLOCKS_SYNCHRONIZING_DEFAULT_COUNT
+  , "How many blocks to sync at once during chain synchronization (0 = adaptive)."
+  , 0
   };
   const command_line::arg_descriptor<std::string> arg_check_updates = {
     "check-updates"

src/cryptonote_basic/connection_context.h

@@ -46,6 +46,7 @@ namespace cryptonote
     {
       state_before_handshake = 0, //default state
       state_synchronizing,
+      state_standby,
       state_idle,
       state_normal
     };
@@ -69,6 +70,8 @@ namespace cryptonote
       return "state_before_handshake";
     case cryptonote_connection_context::state_synchronizing:
       return "state_synchronizing";
+    case cryptonote_connection_context::state_standby:
+      return "state_standby";
     case cryptonote_connection_context::state_idle:
       return "state_idle";
     case cryptonote_connection_context::state_normal:

src/cryptonote_config.h

@@ -89,6 +89,7 @@
 
 
 #define BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT          10000  //by default, blocks ids count in synchronizing
+#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4       100    //by default, blocks count in blocks downloading
 #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT              20     //by default, blocks count in blocks downloading
 #define CRYPTONOTE_PROTOCOL_HOP_RELAX_COUNT             3      //value of hop, after which we use only announce of new block
 

src/cryptonote_core/cryptonote_core.cpp

@@ -421,8 +421,6 @@ namespace cryptonote
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage");
 
     block_sync_size = command_line::get_arg(vm, command_line::arg_block_sync_size);
-    if (block_sync_size == 0)
-      block_sync_size = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
 
     MGINFO("Loading checkpoints");
 
@@ -788,6 +786,16 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  size_t core::get_block_sync_size(uint64_t height) const
+  {
+    static const uint64_t quick_height = m_testnet ? 801219 : 1220516;
+    if (block_sync_size > 0)
+      return block_sync_size;
+    if (height >= quick_height)
+      return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
+    return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4;
+  }
+  //-----------------------------------------------------------------------------------------------
   std::pair<uint64_t, uint64_t> core::get_coinbase_tx_sum(const uint64_t start_offset, const size_t count)
   {
     uint64_t emission_amount = 0;

src/cryptonote_core/cryptonote_core.h

@@ -698,7 +698,7 @@ namespace cryptonote
       *
       * @return the number of blocks to sync in one go
       */
-     size_t get_block_sync_size() const { return block_sync_size; }
+     size_t get_block_sync_size(uint64_t height) const;
 
      /**
       * @brief get the sum of coinbase tx amounts between blocks

src/cryptonote_protocol/block_queue.cpp

@@ -334,6 +334,16 @@ crypto::hash block_queue::get_last_known_hash(const boost::uuids::uuid &connecti
   return hash;
 }
 
+bool block_queue::has_spans(const boost::uuids::uuid &connection_id) const
+{
+  for (const auto &span: blocks)
+  {
+    if (span.connection_id == connection_id)
+      return true;
+  }
+  return false;
+}
+
 float block_queue::get_speed(const boost::uuids::uuid &connection_id) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);

src/cryptonote_protocol/block_queue.h

@@ -86,6 +86,7 @@ namespace cryptonote
     size_t get_num_filled_spans_prefix() const;
     size_t get_num_filled_spans() const;
     crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const;
+    bool has_spans(const boost::uuids::uuid &connection_id) const;
     float get_speed(const boost::uuids::uuid &connection_id) const;
     bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const;
     bool requested(const crypto::hash &hash) const;

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -38,6 +38,7 @@
 #include <string>
 #include <ctime>
 
+#include "math_helper.h"
 #include "storages/levin_abstract_invoke2.h"
 #include "warnings.h"
 #include "cryptonote_protocol_defs.h"
@@ -131,6 +132,7 @@ namespace cryptonote
     bool on_connection_synchronized();
     bool should_download_next_span(cryptonote_connection_context& context) const;
     void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
+    bool kick_idle_peers();
 
     t_core& m_core;
 
@@ -141,6 +143,7 @@ namespace cryptonote
     std::atomic<bool> m_stopping;
     boost::mutex m_sync_lock;
     block_queue m_block_queue;
+    epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker;
 
     boost::mutex m_buffer_mutex;
     double get_avg_block_size();

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -51,6 +51,7 @@
 #define BLOCK_QUEUE_NBLOCKS_THRESHOLD 10 // chunks of N blocks
 #define BLOCK_QUEUE_SIZE_THRESHOLD (100*1024*1024) // MB
 #define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (5 * 1000000) // microseconds
+#define IDLE_PEER_KICK_TIME (45 * 1000000) // microseconds
 
 namespace cryptonote
 {
@@ -1130,10 +1131,33 @@ skip:
   template<class t_core>
   bool t_cryptonote_protocol_handler<t_core>::on_idle()
   {
+    m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::kick_idle_peers, this));
     return m_core.on_idle();
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
+  bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers()
+  {
+    MTRACE("Checking for idle peers...");
+    m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool
+    {
+      if (context.m_state == cryptonote_connection_context::state_synchronizing)
+      {
+        const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
+        const boost::posix_time::time_duration dt = now - context.m_last_request_time;
+        if (dt.total_microseconds() > IDLE_PEER_KICK_TIME)
+        {
+          MINFO(context << " kicking idle peer");
+          ++context.m_callback_request_count;
+          m_p2p->request_callback(context);
+        }
+      }
+      return true;
+    });
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------
+  template<class t_core>
   int t_cryptonote_protocol_handler<t_core>::handle_request_chain(int command, NOTIFY_REQUEST_CHAIN::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_CHAIN (" << arg.block_ids.size() << " blocks");
@@ -1244,6 +1268,7 @@ skip:
         {
           LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", pausing");
           first = false;
+          context.m_state = cryptonote_connection_context::state_standby;
         }
         for (size_t n = 0; n < 50; ++n)
         {
@@ -1252,6 +1277,7 @@ skip:
           boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
         }
       }
+      context.m_state = cryptonote_connection_context::state_synchronizing;
     }
 
     MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " << m_core.get_current_blockchain_height());
@@ -1261,7 +1287,7 @@ skip:
       NOTIFY_REQUEST_GET_OBJECTS::request req;
       bool is_next = false;
       size_t count = 0;
-      const size_t count_limit = m_core.get_block_sync_size();
+      const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height());
       std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0);
       if (force_next_span)
       {
@@ -1403,6 +1429,7 @@ skip:
       //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
       //LOG_PRINT_CCONTEXT_L1("r = " << 200);
 
+      context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
       LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain);
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
     }else

tests/core_proxy/core_proxy.h

@@ -88,7 +88,7 @@ namespace tests
     bool prepare_handle_incoming_blocks(const std::list<cryptonote::block_complete_entry>  &blocks) { return true; }
     bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; }
     uint64_t get_target_blockchain_height() const { return 1; }
-    size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
+    size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
     virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {}
     bool get_testnet() const { return false; }
     bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; }

tests/unit_tests/ban.cpp

@@ -65,7 +65,7 @@ public:
   bool prepare_handle_incoming_blocks(const std::list<cryptonote::block_complete_entry>  &blocks) { return true; }
   bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; }
   uint64_t get_target_blockchain_height() const { return 1; }
-  size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
+  size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
   virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {}
   bool get_testnet() const { return false; }
   bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; }