wallet: background sync with just the view key

- When background syncing, the wallet wipes the spend key
from memory and processes all new transactions. The wallet saves
all receives, spends, and "plausible" spends of receives the
wallet does not know key images for.
- When background sync disabled, the wallet processes all
background synced txs and then clears the background sync cache.
- Adding "plausible" spends to the background sync cache ensures
that the wallet does not need to query the daemon to see if any
received outputs were spent while background sync was enabled.
This would harm privacy especially for users of 3rd party daemons.
- To enable the feature in the CLI wallet, the user can set
background-sync to reuse-wallet-password or
custom-background-password and the wallet automatically syncs in
the background when the wallet locks, then processes all
background synced txs when the wallet is unlocked.
- The custom-background-password option enables the user to
open a distinct background wallet that only has a view key saved
and can be opened/closed/synced separately from the main wallet.
When the main wallet opens, it processes the background wallet's
cache.
- To enable the feature in the RPC wallet, there is a new
`/setup_background_sync` endpoint.
- HW, multsig and view-only wallets cannot background sync.

src/cryptonote_basic/account.cpp

@@ -152,6 +152,17 @@ DISABLE_VS_WARNINGS(4244 4345)
     m_keys.m_multisig_keys.clear();
   }
   //-----------------------------------------------------------------
+  void account_base::set_spend_key(const crypto::secret_key& spend_secret_key)
+  {
+    // make sure derived spend public key matches saved public spend key
+    crypto::public_key spend_public_key;
+    crypto::secret_key_to_public_key(spend_secret_key, spend_public_key);
+    CHECK_AND_ASSERT_THROW_MES(m_keys.m_account_address.m_spend_public_key == spend_public_key,
+        "Unexpected derived public spend key");
+
+    m_keys.m_spend_secret_key = spend_secret_key;
+  }
+  //-----------------------------------------------------------------
   crypto::secret_key account_base::generate(const crypto::secret_key& recovery_key, bool recover, bool two_random)
   {
     crypto::secret_key first = generate_keys(m_keys.m_account_address.m_spend_public_key, m_keys.m_spend_secret_key, recovery_key, recover);

src/cryptonote_basic/account.h

@@ -95,6 +95,7 @@ namespace cryptonote
     bool store(const std::string& file_path);
 
     void forget_spend_key();
+    void set_spend_key(const crypto::secret_key& spend_secret_key);
     const std::vector<crypto::secret_key> &get_multisig_keys() const { return m_keys.m_multisig_keys; }
 
     void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); }

src/cryptonote_config.h

@@ -241,6 +241,8 @@ namespace config
   const unsigned char HASH_KEY_ENCRYPTED_PAYMENT_ID = 0x8d;
   const unsigned char HASH_KEY_WALLET = 0x8c;
   const unsigned char HASH_KEY_WALLET_CACHE = 0x8d;
+  const unsigned char HASH_KEY_BACKGROUND_CACHE = 0x8e;
+  const unsigned char HASH_KEY_BACKGROUND_KEYS_FILE = 0x8f;
   const unsigned char HASH_KEY_RPC_PAYMENT_NONCE = 0x58;
   const unsigned char HASH_KEY_MEMORY = 'k';
   const unsigned char HASH_KEY_MULTISIG[] = {'M', 'u', 'l', 't' , 'i', 's', 'i', 'g', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };

src/simplewallet/simplewallet.cpp

@@ -148,6 +148,17 @@ typedef cryptonote::simple_wallet sw;
     } \
   } while(0)
 
+#define CHECK_IF_BACKGROUND_SYNCING(msg) \
+  do \
+  { \
+    if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing()) \
+    { \
+      std::string type = m_wallet->is_background_wallet() ? "background wallet" : "background syncing wallet"; \
+      fail_msg_writer() << boost::format(tr("%s %s")) % type % msg; \
+      return false; \
+    } \
+  } while (0)
+
 static std::string get_human_readable_timespan(std::chrono::seconds seconds);
 static std::string get_human_readable_timespan(uint64_t seconds);
 
@@ -314,7 +325,7 @@ namespace
     auto pwd_container = tools::password_container::prompt(verify, prompt);
     if (!pwd_container)
     {
-      tools::fail_msg_writer() << sw::tr("failed to read wallet password");
+      tools::fail_msg_writer() << sw::tr("failed to read password");
     }
     return pwd_container;
   }
@@ -324,6 +335,11 @@ namespace
     return password_prompter(verify ? sw::tr("Enter a new password for the wallet") : sw::tr("Wallet password"), verify);
   }
 
+  boost::optional<tools::password_container> background_sync_cache_password_prompter(bool verify)
+  {
+    return password_prompter(verify ? sw::tr("Enter a custom password for the background sync cache") : sw::tr("Background sync cache password"), verify);
+  }
+
   inline std::string interpret_rpc_response(bool ok, const std::string& status)
   {
     std::string err;
@@ -441,6 +457,41 @@ namespace
     return "invalid";
   }
 
+  const struct
+  {
+    const char *name;
+    tools::wallet2::BackgroundSyncType background_sync_type;
+  } background_sync_type_names[] =
+  {
+    { "off", tools::wallet2::BackgroundSyncOff },
+    { "reuse-wallet-password", tools::wallet2::BackgroundSyncReusePassword },
+    { "custom-background-password", tools::wallet2::BackgroundSyncCustomPassword },
+  };
+
+  bool parse_background_sync_type(const std::string &s, tools::wallet2::BackgroundSyncType &background_sync_type)
+  {
+    for (size_t n = 0; n < sizeof(background_sync_type_names) / sizeof(background_sync_type_names[0]); ++n)
+    {
+      if (s == background_sync_type_names[n].name)
+      {
+        background_sync_type = background_sync_type_names[n].background_sync_type;
+        return true;
+      }
+    }
+    fail_msg_writer() << cryptonote::simple_wallet::tr("failed to parse background sync type");
+    return false;
+  }
+
+  std::string get_background_sync_type_name(tools::wallet2::BackgroundSyncType type)
+  {
+    for (size_t n = 0; n < sizeof(background_sync_type_names) / sizeof(background_sync_type_names[0]); ++n)
+    {
+      if (type == background_sync_type_names[n].background_sync_type)
+        return background_sync_type_names[n].name;
+    }
+    return "invalid";
+  }
+
   std::string get_version_string(uint32_t version)
   {
     return boost::lexical_cast<std::string>(version >> 16) + "." + boost::lexical_cast<std::string>(version & 0xffff);
@@ -793,6 +844,7 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto
     fail_msg_writer() << tr("wallet is watch-only and has no spend key");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("has no spend key");
   // don't log
   PAUSE_READLINE();
   if (m_wallet->key_on_device()) {
@@ -823,6 +875,7 @@ bool simple_wallet::print_seed(bool encrypted)
     fail_msg_writer() << tr("wallet is watch-only and has no seed");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("has no seed");
 
   const multisig::multisig_account_status ms_status{m_wallet->get_multisig_status()};
   if (ms_status.multisig_is_active)
@@ -900,6 +953,7 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s
     fail_msg_writer() << tr("wallet is watch-only and has no seed");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("has no seed");
 
   epee::wipeable_string password;
   {
@@ -1046,6 +1100,7 @@ bool simple_wallet::prepare_multisig_main(const std::vector<std::string> &args,
     fail_msg_writer() << tr("wallet is watch-only and cannot be made multisig");
     return false;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot be made multisig");
 
   if(m_wallet->get_num_transfer_details())
   {
@@ -2105,6 +2160,7 @@ bool simple_wallet::save_known_rings(const std::vector<std::string> &args)
 
 bool simple_wallet::freeze_thaw(const std::vector<std::string> &args, bool freeze)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot freeze/thaw");
   if (args.empty())
   {
     fail_msg_writer() << boost::format(tr("usage: %s <key_image>|<pubkey>")) % (freeze ? "freeze" : "thaw");
@@ -2144,6 +2200,7 @@ bool simple_wallet::thaw(const std::vector<std::string> &args)
 
 bool simple_wallet::frozen(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot see frozen key images");
   if (args.empty())
   {
     size_t ntd = m_wallet->get_num_transfer_details();
@@ -2794,6 +2851,57 @@ bool simple_wallet::set_track_uses(const std::vector<std::string> &args/* = std:
   return true;
 }
 
+bool simple_wallet::setup_background_sync(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  if (m_wallet->get_multisig_status().multisig_is_active)
+  {
+    fail_msg_writer() << tr("background sync not implemented for multisig wallet");
+    return true;
+  }
+  if (m_wallet->watch_only())
+  {
+    fail_msg_writer() << tr("background sync not implemented for watch only wallet");
+    return true;
+  }
+  if (m_wallet->key_on_device())
+  {
+    fail_msg_writer() << tr("command not supported by HW wallet");
+    return true;
+  }
+
+  tools::wallet2::BackgroundSyncType background_sync_type;
+  if (!parse_background_sync_type(args[1], background_sync_type))
+  {
+    fail_msg_writer() << tr("invalid option");
+    return true;
+  }
+
+  const auto pwd_container = get_and_verify_password();
+  if (!pwd_container)
+    return true;
+
+  try
+  {
+    boost::optional<epee::wipeable_string> background_cache_password = boost::none;
+    if (background_sync_type == tools::wallet2::BackgroundSyncCustomPassword)
+    {
+      const auto background_pwd_container = background_sync_cache_password_prompter(true);
+      if (!background_pwd_container)
+        return true;
+      background_cache_password = background_pwd_container->password();
+    }
+
+    LOCK_IDLE_SCOPE();
+    m_wallet->setup_background_sync(background_sync_type, pwd_container->password(), background_cache_password);
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Error setting background sync type: ") << e.what();
+  }
+
+  return true;
+}
+
 bool simple_wallet::set_show_wallet_name_when_locked(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   const auto pwd_container = get_and_verify_password();
@@ -3026,6 +3134,7 @@ bool simple_wallet::apropos(const std::vector<std::string> &args)
 
 bool simple_wallet::scan_tx(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot scan tx");
   if (args.empty())
   {
     PRINT_USAGE(USAGE_SCAN_TX);
@@ -3243,6 +3352,8 @@ simple_wallet::simple_wallet()
                                   "  Ignore outputs of amount below this threshold when spending.\n "
                                   "track-uses <1|0>\n "
                                   "  Whether to keep track of owned outputs uses.\n "
+                                  "background-sync <off|reuse-wallet-password|custom-background-password>\n "
+                                  "  Set this to enable scanning in the background with just the view key while the wallet is locked.\n "
                                   "setup-background-mining <1|0>\n "
                                   "  Whether to enable background mining. Set this to support the network and to get a chance to receive new monero.\n "
                                   "device-name <device_name[:device_spec]>\n "
@@ -3645,6 +3756,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     success_msg_writer() << "ignore-outputs-above = " << cryptonote::print_money(m_wallet->ignore_outputs_above());
     success_msg_writer() << "ignore-outputs-below = " << cryptonote::print_money(m_wallet->ignore_outputs_below());
     success_msg_writer() << "track-uses = " << m_wallet->track_uses();
+    success_msg_writer() << "background-sync = " << get_background_sync_type_name(m_wallet->background_sync_type());
     success_msg_writer() << "setup-background-mining = " << setup_background_mining_string;
     success_msg_writer() << "device-name = " << m_wallet->device_name();
     success_msg_writer() << "export-format = " << (m_wallet->export_format() == tools::wallet2::ExportFormat::Ascii ? "ascii" : "binary");
@@ -3660,6 +3772,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
   }
   else
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot change wallet settings");
 
 #define CHECK_SIMPLE_VARIABLE(name, f, help) do \
   if (args[0] == name) { \
@@ -3713,6 +3826,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     CHECK_SIMPLE_VARIABLE("ignore-outputs-above", set_ignore_outputs_above, tr("amount"));
     CHECK_SIMPLE_VARIABLE("ignore-outputs-below", set_ignore_outputs_below, tr("amount"));
     CHECK_SIMPLE_VARIABLE("track-uses", set_track_uses, tr("0 or 1"));
+    CHECK_SIMPLE_VARIABLE("background-sync", setup_background_sync, tr("off (default); reuse-wallet-password (reuse the wallet password to encrypt the background cache); custom-background-password (use a custom background password to encrypt the background cache)"));
     CHECK_SIMPLE_VARIABLE("show-wallet-name-when-locked", set_show_wallet_name_when_locked, tr("1 or 0"));
     CHECK_SIMPLE_VARIABLE("inactivity-lock-timeout", set_inactivity_lock_timeout, tr("unsigned integer (seconds, 0 to disable)"));
     CHECK_SIMPLE_VARIABLE("setup-background-mining", set_setup_background_mining, tr("1/yes or 0/no"));
@@ -4653,7 +4767,10 @@ std::string simple_wallet::get_mnemonic_language()
 //----------------------------------------------------------------------------------------------------
 boost::optional<tools::password_container> simple_wallet::get_and_verify_password() const
 {
-  auto pwd_container = default_password_prompter(m_wallet_file.empty());
+  const bool verify = m_wallet_file.empty();
+  auto pwd_container = (m_wallet->is_background_wallet() && m_wallet->background_sync_type() == tools::wallet2::BackgroundSyncCustomPassword)
+    ? background_sync_cache_password_prompter(verify)
+    : default_password_prompter(verify);
   if (!pwd_container)
     return boost::none;
 
@@ -4956,6 +5073,8 @@ boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::p
       prefix = tr("Opened watch-only wallet");
     else if (ms_status.multisig_is_active)
       prefix = (boost::format(tr("Opened %u/%u multisig wallet%s")) % ms_status.threshold % ms_status.total % (ms_status.is_ready ? "" : " (not yet finalized)")).str();
+    else if (m_wallet->is_background_wallet())
+      prefix = tr("Opened background wallet");
     else
       prefix = tr("Opened wallet");
     message_writer(console_color_white, true) <<
@@ -5163,6 +5282,10 @@ void simple_wallet::stop_background_mining()
 //----------------------------------------------------------------------------------------------------
 void simple_wallet::check_background_mining(const epee::wipeable_string &password)
 {
+  // Background mining can be toggled from the main wallet
+  if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing())
+    return;
+
   tools::wallet2::BackgroundMiningSetupType setup = m_wallet->setup_background_mining();
   if (setup == tools::wallet2::BackgroundMiningNo)
   {
@@ -5978,6 +6101,7 @@ bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::rescan_spent(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot rescan spent");
   if (!m_wallet->is_trusted_daemon())
   {
     fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon");
@@ -6233,10 +6357,27 @@ void simple_wallet::check_for_inactivity_lock(bool user)
           "                ||     ||" << std::endl <<
           "" << std::endl;
     }
+
+    bool started_background_sync = false;
+    if (!m_wallet->is_background_wallet() &&
+        m_wallet->background_sync_type() != tools::wallet2::BackgroundSyncOff)
+    {
+      LOCK_IDLE_SCOPE();
+      m_wallet->start_background_sync();
+      started_background_sync = true;
+    }
+
     while (1)
     {
       const char *inactivity_msg = user ? "" : tr("Locked due to inactivity.");
-      tools::msg_writer() << inactivity_msg << (inactivity_msg[0] ? " " : "") << tr("The wallet password is required to unlock the console.");
+      tools::msg_writer() << inactivity_msg << (inactivity_msg[0] ? " " : "") << (
+        (m_wallet->is_background_wallet() && m_wallet->background_sync_type() == tools::wallet2::BackgroundSyncCustomPassword)
+            ? tr("The background password is required to unlock the console.")
+            : tr("The wallet password is required to unlock the console.")
+      );
+
+      if (m_wallet->is_background_syncing())
+        tools::msg_writer() << tr("\nSyncing in the background while locked...") << std::endl;
 
       const bool show_wallet_name = m_wallet->show_wallet_name_when_locked();
       if (show_wallet_name)
@@ -6249,8 +6390,16 @@ void simple_wallet::check_for_inactivity_lock(bool user)
       }
       try
       {
-        if (get_and_verify_password())
+        const auto pwd_container = get_and_verify_password();
+        if (pwd_container)
+        {
+          if (started_background_sync)
+          {
+            LOCK_IDLE_SCOPE();
+            m_wallet->stop_background_sync(pwd_container->password());
+          }
           break;
+        }
       }
       catch (...) { /* do nothing, just let the loop loop */ }
     }
@@ -6277,6 +6426,7 @@ bool simple_wallet::on_command(bool (simple_wallet::*cmd)(const std::vector<std:
 bool simple_wallet::transfer_main(const std::vector<std::string> &args_, bool called_by_mms)
 {
 //  "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"
+  CHECK_IF_BACKGROUND_SYNCING("cannot transfer");
   if (!try_connect_to_daemon())
     return false;
 
@@ -6690,6 +6840,7 @@ bool simple_wallet::transfer_main(const std::vector<std::string> &args_, bool ca
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::transfer(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot transfer");
   if (args_.size() < 1)
   {
     PRINT_USAGE(USAGE_TRANSFER);
@@ -6702,6 +6853,7 @@ bool simple_wallet::transfer(const std::vector<std::string> &args_)
 
 bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   if (!try_connect_to_daemon())
     return true;
 
@@ -6809,6 +6961,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_main(uint32_t account, uint64_t below, const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   auto print_usage = [this, account, below]()
   {
     if (below)
@@ -7090,6 +7243,7 @@ bool simple_wallet::sweep_main(uint32_t account, uint64_t below, const std::vect
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_single(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   if (!try_connect_to_daemon())
     return true;
 
@@ -7328,12 +7482,14 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_all(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   sweep_main(m_current_subaddress_account, 0, args_);
   return true;
 }
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_account(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   auto local_args = args_;
   if (local_args.empty())
   {
@@ -7354,6 +7510,7 @@ bool simple_wallet::sweep_account(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_below(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sweep");
   uint64_t below = 0;
   if (args_.size() < 1)
   {
@@ -7372,6 +7529,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::donate(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot donate");
   std::vector<std::string> local_args = args_;
   if(local_args.empty() || local_args.size() > 5)
   {
@@ -7433,6 +7591,7 @@ bool simple_wallet::donate(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, const std::function<const tools::wallet2::tx_construction_data&(size_t)> &get_tx, const std::string &extra_message)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot load tx");
   // gather info to ask the user
   uint64_t amount = 0, amount_to_dests = 0, change = 0;
   size_t min_ring_size = ~0;
@@ -7613,6 +7772,7 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_)
      fail_msg_writer() << tr("This is a watch only wallet");
      return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot sign transfer");
 
   bool export_raw = false;
   std::string unsigned_filename = "unsigned_monero_tx";
@@ -7720,6 +7880,8 @@ std::string get_tx_key_stream(crypto::secret_key tx_key, std::vector<crypto::sec
 
 bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get tx key");
+
   std::vector<std::string> local_args = args_;
 
   if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR)
@@ -7760,6 +7922,8 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::set_tx_key(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot set tx key");
+
   std::vector<std::string> local_args = args_;
 
   if(local_args.size() != 2 && local_args.size() != 3) {
@@ -7836,6 +8000,8 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_tx_proof(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get tx proof");
+
   if (args.size() != 2 && args.size() != 3)
   {
     PRINT_USAGE(USAGE_GET_TX_PROOF);
@@ -8042,6 +8208,7 @@ bool simple_wallet::check_tx_proof(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_spend_proof(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get spend proof");
   if (m_wallet->key_on_device())
   {
     fail_msg_writer() << tr("command not supported by HW wallet");
@@ -8126,6 +8293,7 @@ bool simple_wallet::check_spend_proof(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get reserve proof");
   if (m_wallet->key_on_device())
   {
     fail_msg_writer() << tr("command not supported by HW wallet");
@@ -8812,6 +8980,8 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot rescan");
+
   uint64_t start_height = 0;
   ResetType reset_type = ResetSoft;
 
@@ -9036,6 +9206,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector
   if (command == "new")
   {
     // create a new account and switch to it
+    CHECK_IF_BACKGROUND_SYNCING("cannot create new account");
     std::string label = boost::join(local_args, " ");
     if (label.empty())
       label = tr("(Untitled account)");
@@ -9066,6 +9237,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector
   else if (command == "label" && local_args.size() >= 1)
   {
     // set label of the specified account
+    CHECK_IF_BACKGROUND_SYNCING("cannot modify account");
     uint32_t index_major;
     if (!epee::string_tools::get_xtype_from_string(index_major, local_args[0]))
     {
@@ -9087,6 +9259,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector
   }
   else if (command == "tag" && local_args.size() >= 2)
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot modify account");
     const std::string tag = local_args[0];
     std::set<uint32_t> account_indices;
     for (size_t i = 1; i < local_args.size(); ++i)
@@ -9111,6 +9284,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector
   }
   else if (command == "untag" && local_args.size() >= 1)
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot modify account");
     std::set<uint32_t> account_indices;
     for (size_t i = 0; i < local_args.size(); ++i)
     {
@@ -9134,6 +9308,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector
   }
   else if (command == "tag_description" && local_args.size() >= 1)
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot modify account");
     const std::string tag = local_args[0];
     std::string description;
     if (local_args.size() > 1)
@@ -9251,6 +9426,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
   }
   else if (local_args[0] == "new")
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot add address");
     local_args.erase(local_args.begin());
     std::string label;
     if (local_args.size() > 0)
@@ -9263,6 +9439,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
   }
   else if (local_args[0] == "mnew")
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot add addresses");
     local_args.erase(local_args.begin());
     if (local_args.size() != 1)
     {
@@ -9288,6 +9465,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
   }
   else if (local_args[0] == "one-off")
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot add address");
     local_args.erase(local_args.begin());
     std::string label;
     if (local_args.size() != 2)
@@ -9306,6 +9484,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
   }
   else if (local_args.size() >= 2 && local_args[0] == "label")
   {
+    CHECK_IF_BACKGROUND_SYNCING("cannot modify address");
     if (!epee::string_tools::get_xtype_from_string(index, local_args[1]))
     {
       fail_msg_writer() << tr("failed to parse index: ") << local_args[1];
@@ -9452,6 +9631,8 @@ bool simple_wallet::print_integrated_address(const std::vector<std::string> &arg
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get address book");
+
   if (args.size() == 0)
   {
   }
@@ -9512,6 +9693,8 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::set_tx_note(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot set tx note");
+
   if (args.size() == 0)
   {
     PRINT_USAGE(USAGE_SET_TX_NOTE);
@@ -9540,6 +9723,8 @@ bool simple_wallet::set_tx_note(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_tx_note(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get tx note");
+
   if (args.size() != 1)
   {
     PRINT_USAGE(USAGE_GET_TX_NOTE);
@@ -9565,6 +9750,8 @@ bool simple_wallet::get_tx_note(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::set_description(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot set description");
+
   // 0 arguments allowed, for setting the description to empty string
 
   std::string description = "";
@@ -9581,6 +9768,8 @@ bool simple_wallet::set_description(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_description(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot get description");
+
   if (args.size() != 0)
   {
     PRINT_USAGE(USAGE_GET_DESCRIPTION);
@@ -9639,6 +9828,8 @@ bool simple_wallet::wallet_info(const std::vector<std::string> &args)
     type = tr("Watch only");
   else if (ms_status.multisig_is_active)
     type = (boost::format(tr("%u/%u multisig%s")) % ms_status.threshold % ms_status.total % (ms_status.is_ready ? "" : " (not yet finalized)")).str();
+  else if (m_wallet->is_background_wallet())
+    type = tr("Background wallet");
   else
     type = tr("Normal");
   message_writer() << tr("Type: ") << type;
@@ -9650,6 +9841,7 @@ bool simple_wallet::wallet_info(const std::vector<std::string> &args)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sign(const std::vector<std::string> &args)
 {
+  CHECK_IF_BACKGROUND_SYNCING("cannot sign");
   if (m_wallet->key_on_device())
   {
     fail_msg_writer() << tr("command not supported by HW wallet");
@@ -9757,6 +9949,7 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args_)
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot export key images");
   auto args = args_;
 
   if (m_wallet->watch_only())
@@ -9810,6 +10003,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args)
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot import key images");
   if (!m_wallet->is_trusted_daemon())
   {
     fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon");
@@ -9918,6 +10112,7 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args_)
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot export outputs");
   auto args = args_;
 
   bool all = false;
@@ -9967,6 +10162,7 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args)
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;
   }
+  CHECK_IF_BACKGROUND_SYNCING("cannot import outputs");
   if (args.size() != 1)
   {
     PRINT_USAGE(USAGE_IMPORT_OUTPUTS);

src/simplewallet/simplewallet.h

@@ -147,6 +147,7 @@ namespace cryptonote
     bool set_ignore_outputs_above(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_ignore_outputs_below(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_track_uses(const std::vector<std::string> &args = std::vector<std::string>());
+    bool setup_background_sync(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_show_wallet_name_when_locked(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_inactivity_lock_timeout(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_setup_background_mining(const std::vector<std::string> &args = std::vector<std::string>());

src/wallet/api/wallet.cpp

@@ -58,6 +58,40 @@ using namespace cryptonote;
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "WalletAPI"
 
+#define LOCK_REFRESH() \
+    bool refresh_enabled = m_refreshEnabled; \
+    m_refreshEnabled = false; \
+    m_wallet->stop(); \
+    m_refreshCV.notify_one(); \
+    boost::mutex::scoped_lock lock(m_refreshMutex); \
+    boost::mutex::scoped_lock lock2(m_refreshMutex2); \
+    epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \
+        /* m_refreshMutex's still locked here */ \
+        if (refresh_enabled) \
+            startRefresh(); \
+    })
+
+#define PRE_VALIDATE_BACKGROUND_SYNC() \
+  do \
+  { \
+    clearStatus(); \
+    if (m_wallet->key_on_device()) \
+    { \
+        setStatusError(tr("HW wallet cannot use background sync")); \
+        return false; \
+    } \
+    if (m_wallet->watch_only()) \
+    { \
+        setStatusError(tr("View only wallet cannot use background sync")); \
+        return false; \
+    } \
+    if (m_wallet->get_multisig_status().multisig_is_active) \
+    { \
+        setStatusError(tr("Multisig wallet cannot use background sync")); \
+        return false; \
+    } \
+  } while (0)
+
 namespace Monero {
 
 namespace {
@@ -766,6 +800,8 @@ bool WalletImpl::close(bool store)
 
 std::string WalletImpl::seed(const std::string& seed_offset) const
 {
+    if (checkBackgroundSync("cannot get seed"))
+        return std::string();
     epee::wipeable_string seed;
     if (m_wallet)
         m_wallet->get_seed(seed, seed_offset);
@@ -779,6 +815,8 @@ std::string WalletImpl::getSeedLanguage() const
 
 void WalletImpl::setSeedLanguage(const std::string &arg)
 {
+    if (checkBackgroundSync("cannot set seed language"))
+        return;
     m_wallet->set_seed_language(arg);
 }
 
@@ -802,6 +840,8 @@ void WalletImpl::statusWithErrorString(int& status, std::string& errorString) co
 
 bool WalletImpl::setPassword(const std::string &password)
 {
+    if (checkBackgroundSync("cannot change password"))
+        return false;
     clearStatus();
     try {
         m_wallet->change_password(m_wallet->get_wallet_file(), m_password, password);
@@ -931,6 +971,8 @@ bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transact
 
 void WalletImpl::setRefreshFromBlockHeight(uint64_t refresh_from_block_height)
 {
+    if (checkBackgroundSync("cannot change refresh height"))
+        return;
     m_wallet->set_refresh_from_block_height(refresh_from_block_height);
 }
 
@@ -1039,6 +1081,8 @@ void WalletImpl::refreshAsync()
 
 bool WalletImpl::rescanBlockchain()
 {
+    if (checkBackgroundSync("cannot rescan blockchain"))
+        return false;
     clearStatus();
     m_refreshShouldRescan = true;
     doRefresh();
@@ -1047,6 +1091,8 @@ bool WalletImpl::rescanBlockchain()
 
 void WalletImpl::rescanBlockchainAsync()
 {
+    if (checkBackgroundSync("cannot rescan blockchain"))
+        return;
     m_refreshShouldRescan = true;
     refreshAsync();
 }
@@ -1070,7 +1116,7 @@ int WalletImpl::autoRefreshInterval() const
 UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_filename) {
   clearStatus();
   UnsignedTransactionImpl * transaction = new UnsignedTransactionImpl(*this);
-  if (!m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){
+  if (checkBackgroundSync("cannot load tx") || !m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){
     setStatusError(tr("Failed to load unsigned transactions"));
     transaction->m_status = UnsignedTransaction::Status::Status_Error;
     transaction->m_errorString = errorString();
@@ -1090,6 +1136,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file
 
 bool WalletImpl::submitTransaction(const string &fileName) {
   clearStatus();
+  if (checkBackgroundSync("cannot submit tx"))
+    return false;
   std::unique_ptr<PendingTransactionImpl> transaction(new PendingTransactionImpl(*this));
 
   bool r = m_wallet->load_tx(fileName, transaction->m_pending_tx);
@@ -1113,6 +1161,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all)
     setStatusError(tr("Wallet is view only"));
     return false;
   }
+  if (checkBackgroundSync("cannot export key images"))
+    return false;
   
   try
   {
@@ -1133,6 +1183,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all)
 
 bool WalletImpl::importKeyImages(const string &filename)
 {
+  if (checkBackgroundSync("cannot import key images"))
+    return false;
   if (!trustedDaemon()) {
     setStatusError(tr("Key images can only be imported with a trusted daemon"));
     return false;
@@ -1156,6 +1208,8 @@ bool WalletImpl::importKeyImages(const string &filename)
 
 bool WalletImpl::exportOutputs(const string &filename, bool all)
 {
+    if (checkBackgroundSync("cannot export outputs"))
+        return false;
     if (m_wallet->key_on_device())
     {
         setStatusError(string(tr("Not supported on HW wallets.")) + filename);
@@ -1186,6 +1240,8 @@ bool WalletImpl::exportOutputs(const string &filename, bool all)
 
 bool WalletImpl::importOutputs(const string &filename)
 {
+    if (checkBackgroundSync("cannot import outputs"))
+        return false;
     if (m_wallet->key_on_device())
     {
         setStatusError(string(tr("Not supported on HW wallets.")) + filename);
@@ -1218,6 +1274,8 @@ bool WalletImpl::importOutputs(const string &filename)
 
 bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
 {
+    if (checkBackgroundSync("cannot scan transactions"))
+        return false;
     if (txids.empty())
     {
         setStatusError(string(tr("Failed to scan transactions: no transaction ids provided.")));
@@ -1256,8 +1314,86 @@ bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
     return true;
 }
 
+bool WalletImpl::setupBackgroundSync(const Wallet::BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password)
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+
+        tools::wallet2::BackgroundSyncType bgs_type;
+        switch (background_sync_type)
+        {
+            case Wallet::BackgroundSync_Off: bgs_type = tools::wallet2::BackgroundSyncOff; break;
+            case Wallet::BackgroundSync_ReusePassword: bgs_type = tools::wallet2::BackgroundSyncReusePassword; break;
+            case Wallet::BackgroundSync_CustomPassword: bgs_type = tools::wallet2::BackgroundSyncCustomPassword; break;
+            default: setStatusError(tr("Unknown background sync type")); return false;
+        }
+
+        boost::optional<epee::wipeable_string> bgc_password = background_cache_password
+            ? boost::optional<epee::wipeable_string>(*background_cache_password)
+            : boost::none;
+
+        LOCK_REFRESH();
+        m_wallet->setup_background_sync(bgs_type, wallet_password, bgc_password);
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to setup background sync: " << e.what());
+        setStatusError(string(tr("Failed to setup background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
+Wallet::BackgroundSyncType WalletImpl::getBackgroundSyncType() const
+{
+    switch (m_wallet->background_sync_type())
+    {
+        case tools::wallet2::BackgroundSyncOff: return Wallet::BackgroundSync_Off;
+        case tools::wallet2::BackgroundSyncReusePassword: return Wallet::BackgroundSync_ReusePassword;
+        case tools::wallet2::BackgroundSyncCustomPassword: return Wallet::BackgroundSync_CustomPassword;
+        default: setStatusError(tr("Unknown background sync type")); return Wallet::BackgroundSync_Off;
+    }
+}
+
+bool WalletImpl::startBackgroundSync()
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+        LOCK_REFRESH();
+        m_wallet->start_background_sync();
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to start background sync: " << e.what());
+        setStatusError(string(tr("Failed to start background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
+bool WalletImpl::stopBackgroundSync(const std::string &wallet_password)
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+        LOCK_REFRESH();
+        m_wallet->stop_background_sync(epee::wipeable_string(wallet_password));
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to stop background sync: " << e.what());
+        setStatusError(string(tr("Failed to stop background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
 void WalletImpl::addSubaddressAccount(const std::string& label)
 {
+    if (checkBackgroundSync("cannot add account"))
+        return;
     m_wallet->add_subaddress_account(label);
 }
 size_t WalletImpl::numSubaddressAccounts() const
@@ -1270,10 +1406,14 @@ size_t WalletImpl::numSubaddresses(uint32_t accountIndex) const
 }
 void WalletImpl::addSubaddress(uint32_t accountIndex, const std::string& label)
 {
+    if (checkBackgroundSync("cannot add subbaddress"))
+        return;
     m_wallet->add_subaddress(accountIndex, label);
 }
 std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const
 {
+    if (checkBackgroundSync("cannot get subbaddress label"))
+        return "";
     try
     {
         return m_wallet->get_subaddress_label({accountIndex, addressIndex});
@@ -1287,6 +1427,8 @@ std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addre
 }
 void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label)
 {
+    if (checkBackgroundSync("cannot set subbaddress label"))
+        return;
     try
     {
         return m_wallet->set_subaddress_label({accountIndex, addressIndex}, label);
@@ -1300,6 +1442,9 @@ void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex
 
 MultisigState WalletImpl::multisig() const {
     MultisigState state;
+    if (checkBackgroundSync("cannot use multisig"))
+        return state;
+
     const multisig::multisig_account_status ms_status{m_wallet->get_multisig_status()};
 
     state.isMultisig = ms_status.multisig_is_active;
@@ -1312,6 +1457,8 @@ MultisigState WalletImpl::multisig() const {
 }
 
 string WalletImpl::getMultisigInfo() const {
+    if (checkBackgroundSync("cannot use multisig"))
+        return string();
     try {
         clearStatus();
         return m_wallet->get_multisig_first_kex_msg();
@@ -1324,6 +1471,8 @@ string WalletImpl::getMultisigInfo() const {
 }
 
 string WalletImpl::makeMultisig(const vector<string>& info, const uint32_t threshold) {
+    if (checkBackgroundSync("cannot make multisig"))
+        return string();
     try {
         clearStatus();
 
@@ -1464,6 +1613,9 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri
     PendingTransactionImpl * transaction = new PendingTransactionImpl(*this);
 
     do {
+        if (checkBackgroundSync("cannot create transactions"))
+            break;
+
         std::vector<uint8_t> extra;
         std::string extra_nonce;
         vector<cryptonote::tx_destination_entry> dsts;
@@ -1630,6 +1782,9 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction()
     PendingTransactionImpl * transaction = new PendingTransactionImpl(*this);
 
     do {
+        if (checkBackgroundSync("cannot sweep"))
+            break;
+
         try {
             transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions();
             pendingTxPostProcess(transaction);
@@ -1763,11 +1918,15 @@ uint32_t WalletImpl::defaultMixin() const
 
 void WalletImpl::setDefaultMixin(uint32_t arg)
 {
+    if (checkBackgroundSync("cannot set default mixin"))
+        return;
     m_wallet->default_mixin(arg);
 }
 
 bool WalletImpl::setCacheAttribute(const std::string &key, const std::string &val)
 {
+    if (checkBackgroundSync("cannot set cache attribute"))
+        return false;
     m_wallet->set_attribute(key, val);
     return true;
 }
@@ -1781,6 +1940,8 @@ std::string WalletImpl::getCacheAttribute(const std::string &key) const
 
 bool WalletImpl::setUserNote(const std::string &txid, const std::string &note)
 {
+    if (checkBackgroundSync("cannot set user note"))
+        return false;
     cryptonote::blobdata txid_data;
     if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash))
       return false;
@@ -1792,6 +1953,8 @@ bool WalletImpl::setUserNote(const std::string &txid, const std::string &note)
 
 std::string WalletImpl::getUserNote(const std::string &txid) const
 {
+    if (checkBackgroundSync("cannot get user note"))
+        return "";
     cryptonote::blobdata txid_data;
     if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash))
       return "";
@@ -1802,6 +1965,9 @@ std::string WalletImpl::getUserNote(const std::string &txid) const
 
 std::string WalletImpl::getTxKey(const std::string &txid_str) const
 {
+    if (checkBackgroundSync("cannot get tx key"))
+        return "";
+
     crypto::hash txid;
     if(!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1886,6 +2052,9 @@ bool WalletImpl::checkTxKey(const std::string &txid_str, std::string tx_key_str,
 
 std::string WalletImpl::getTxProof(const std::string &txid_str, const std::string &address_str, const std::string &message) const
 {
+    if (checkBackgroundSync("cannot get tx proof"))
+        return "";
+
     crypto::hash txid;
     if (!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1942,6 +2111,9 @@ bool WalletImpl::checkTxProof(const std::string &txid_str, const std::string &ad
 }
 
 std::string WalletImpl::getSpendProof(const std::string &txid_str, const std::string &message) const {
+    if (checkBackgroundSync("cannot get spend proof"))
+        return "";
+
     crypto::hash txid;
     if(!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1984,6 +2156,9 @@ bool WalletImpl::checkSpendProof(const std::string &txid_str, const std::string
 }
 
 std::string WalletImpl::getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const {
+    if (checkBackgroundSync("cannot get reserve proof"))
+        return "";
+
     try
     {
         clearStatus();
@@ -2030,6 +2205,9 @@ bool WalletImpl::checkReserveProof(const std::string &address, const std::string
 
 std::string WalletImpl::signMessage(const std::string &message, const std::string &address)
 {
+    if (checkBackgroundSync("cannot sign message"))
+        return "";
+
     if (address.empty()) {
         return m_wallet->sign(message, tools::wallet2::sign_with_spend_key);
     }
@@ -2156,6 +2334,16 @@ bool WalletImpl::isDeterministic() const
     return m_wallet->is_deterministic();
 }
 
+bool WalletImpl::isBackgroundSyncing() const
+{
+    return m_wallet->is_background_syncing();
+}
+
+bool WalletImpl::isBackgroundWallet() const
+{
+    return m_wallet->is_background_wallet();
+}
+
 void WalletImpl::clearStatus() const
 {
     boost::lock_guard<boost::mutex> l(m_statusMutex);
@@ -2224,9 +2412,7 @@ void WalletImpl::doRefresh()
             if(rescan)
                 m_wallet->rescan_blockchain(false);
             m_wallet->refresh(trustedDaemon());
-            if (!m_synchronized) {
+            m_synchronized = m_wallet->is_synced();
-                m_synchronized = true;
-            }
             // assuming if we have empty history, it wasn't initialized yet
             // for further history changes client need to update history in
             // "on_money_received" and "on_money_sent" callbacks
@@ -2329,6 +2515,24 @@ bool WalletImpl::doInit(const string &daemon_address, const std::string &proxy_a
     return true;
 }
 
+bool WalletImpl::checkBackgroundSync(const std::string &message) const
+{
+    clearStatus();
+    if (m_wallet->is_background_wallet())
+    {
+        LOG_ERROR("Background wallets " + message);
+        setStatusError(tr("Background wallets ") + message);
+        return true;
+    }
+    if (m_wallet->is_background_syncing())
+    {
+        LOG_ERROR(message + " while background syncing");
+        setStatusError(message + tr(" while background syncing. Stop background syncing first."));
+        return true;
+    }
+    return false;
+}
+
 bool WalletImpl::parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error)
 {
     return m_wallet->parse_uri(uri, address, payment_id, amount, tx_description, recipient_name, unknown_parameters, error);
@@ -2347,6 +2551,8 @@ std::string WalletImpl::getDefaultDataDir() const
 bool WalletImpl::rescanSpent()
 {
   clearStatus();
+  if (checkBackgroundSync("cannot rescan spent"))
+    return false;
   if (!trustedDaemon()) {
     setStatusError(tr("Rescan spent can only be used with a trusted daemon"));
     return false;

src/wallet/api/wallet.h

@@ -172,6 +172,13 @@ public:
     bool importOutputs(const std::string &filename) override;
     bool scanTransactions(const std::vector<std::string> &txids) override;
 
+    bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password = optional<std::string>()) override;
+    BackgroundSyncType getBackgroundSyncType() const override;
+    bool startBackgroundSync() override;
+    bool stopBackgroundSync(const std::string &wallet_password) override;
+    bool isBackgroundSyncing() const override;
+    bool isBackgroundWallet() const override;
+
     virtual void disposeTransaction(PendingTransaction * t) override;
     virtual uint64_t estimateTransactionFee(const std::vector<std::pair<std::string, uint64_t>> &destinations,
                                             PendingTransaction::Priority priority) const override;
@@ -238,6 +245,7 @@ private:
     bool isNewWallet() const;
     void pendingTxPostProcess(PendingTransactionImpl * pending);
     bool doInit(const std::string &daemon_address, const std::string &proxy_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false);
+    bool checkBackgroundSync(const std::string &message) const;
 
 private:
     friend class PendingTransactionImpl;
@@ -253,6 +261,10 @@ private:
     mutable boost::mutex m_statusMutex;
     mutable int m_status;
     mutable std::string m_errorString;
+    // TODO: harden password handling in the wallet API, see relevant discussion
+    // https://github.com/monero-project/monero-gui/issues/1537
+    // https://github.com/feather-wallet/feather/issues/72#issuecomment-1405602142
+    // https://github.com/monero-project/monero/pull/8619#issuecomment-1632951461
     std::string m_password;
     std::unique_ptr<TransactionHistoryImpl> m_history;
     std::unique_ptr<Wallet2CallbackImpl> m_wallet2Callback;

src/wallet/api/wallet2_api.h

@@ -446,6 +446,12 @@ struct Wallet
         ConnectionStatus_WrongVersion
     };
 
+    enum BackgroundSyncType {
+        BackgroundSync_Off = 0,
+        BackgroundSync_ReusePassword = 1,
+        BackgroundSync_CustomPassword = 2
+    };
+
     virtual ~Wallet() = 0;
     virtual std::string seed(const std::string& seed_offset = "") const = 0;
     virtual std::string getSeedLanguage() const = 0;
@@ -937,6 +943,42 @@ struct Wallet
      */
     virtual bool scanTransactions(const std::vector<std::string> &txids) = 0;
 
+    /*!
+     * \brief setupBackgroundSync       - setup background sync mode with just a view key
+     * \param background_sync_type      - the mode the wallet background syncs in
+     * \param wallet_password
+     * \param background_cache_password - custom password to encrypt background cache, only needed for custom password background sync type
+     * \return                          - true on success
+     */
+    virtual bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password) = 0;
+
+    /*!
+     * \brief getBackgroundSyncType     - get mode the wallet background syncs in
+     * \return                          - the type, or off if type is unknown
+     */
+    virtual BackgroundSyncType getBackgroundSyncType() const = 0;
+
+    /**
+     * @brief startBackgroundSync - sync the chain in the background with just view key
+     */
+    virtual bool startBackgroundSync() = 0;
+
+    /**
+     * @brief stopBackgroundSync  - bring back spend key and process background synced txs
+     * \param wallet_password
+     */
+    virtual bool stopBackgroundSync(const std::string &wallet_password) = 0;
+
+    /**
+     * @brief isBackgroundSyncing - returns true if the wallet is background syncing
+     */
+    virtual bool isBackgroundSyncing() const = 0;
+
+    /**
+     * @brief isBackgroundWallet - returns true if the wallet is a background wallet
+     */
+    virtual bool isBackgroundWallet() const = 0;
+
     virtual TransactionHistory * history() = 0;
     virtual AddressBook * addressBook() = 0;
     virtual Subaddress * subaddress() = 0;

src/wallet/wallet2.h

@@ -249,6 +249,20 @@ private:
       BackgroundMiningNo = 2,
     };
 
+    enum BackgroundSyncType {
+      BackgroundSyncOff = 0,
+      BackgroundSyncReusePassword = 1,
+      BackgroundSyncCustomPassword = 2,
+    };
+
+    static BackgroundSyncType background_sync_type_from_str(const std::string &background_sync_type_str)
+    {
+      if (background_sync_type_str == "off") return BackgroundSyncOff;
+      if (background_sync_type_str == "reuse-wallet-password") return BackgroundSyncReusePassword;
+      if (background_sync_type_str == "custom-background-password") return BackgroundSyncCustomPassword;
+      throw std::logic_error("Unknown background sync type");
+    };
+
     enum ExportFormat {
       Binary = 0,
       Ascii,
@@ -275,7 +289,12 @@ private:
     //! Just parses variables.
     static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter);
 
-    static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds);
+    static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds)
+    {
+      crypto::secret_key spend_key = crypto::null_skey;
+      return verify_password(keys_file_name, password, no_spend_key, hwdev, kdf_rounds, spend_key);
+    };
+    static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds, crypto::secret_key &spend_key_out);
     static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1);
 
     wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_client_factory>(new net::http::client_factory()));
@@ -785,6 +804,54 @@ private:
       END_SERIALIZE()
     };
 
+    struct background_synced_tx_t
+    {
+      uint64_t index_in_background_sync_data;
+      cryptonote::transaction tx;
+      std::vector<uint64_t> output_indices;
+      uint64_t height;
+      uint64_t block_timestamp;
+      bool double_spend_seen;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        VARINT_FIELD(index_in_background_sync_data)
+
+        // prune tx; don't need to keep signature data
+        if (!tx.serialize_base(ar))
+          return false;
+
+        FIELD(output_indices)
+        VARINT_FIELD(height)
+        VARINT_FIELD(block_timestamp)
+        FIELD(double_spend_seen)
+      END_SERIALIZE()
+    };
+
+    struct background_sync_data_t
+    {
+      bool first_refresh_done = false;
+      uint64_t start_height = 0;
+      std::unordered_map<crypto::hash, background_synced_tx_t> txs;
+
+      // Relevant wallet settings
+      uint64_t wallet_refresh_from_block_height;
+      size_t subaddress_lookahead_major;
+      size_t subaddress_lookahead_minor;
+      RefreshType wallet_refresh_type;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(first_refresh_done)
+        FIELD(start_height)
+        FIELD(txs)
+        FIELD(wallet_refresh_from_block_height)
+        VARINT_FIELD(subaddress_lookahead_major)
+        VARINT_FIELD(subaddress_lookahead_minor)
+        VARINT_FIELD(wallet_refresh_type)
+      END_SERIALIZE()
+    };
+
     typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry;
 
     struct parsed_block
@@ -973,7 +1040,8 @@ private:
     /*!
      * \brief verifies given password is correct for default wallet keys file
      */
-    bool verify_password(const epee::wipeable_string& password);
+    bool verify_password(const epee::wipeable_string& password) {crypto::secret_key key = crypto::null_skey; return verify_password(password, key);};
+    bool verify_password(const epee::wipeable_string& password, crypto::secret_key &spend_key_out);
     cryptonote::account_base& get_account(){return m_account;}
     const cryptonote::account_base& get_account()const{return m_account;}
 
@@ -1060,6 +1128,7 @@ private:
 
     cryptonote::network_type nettype() const { return m_nettype; }
     bool watch_only() const { return m_watch_only; }
+    bool is_background_wallet() const { return m_is_background_wallet; }
     multisig::multisig_account_status get_multisig_status() const;
     bool has_multisig_partial_key_images() const;
     bool has_unknown_key_images() const;
@@ -1269,11 +1338,17 @@ private:
         return;
       }
       a & m_has_ever_refreshed_from_node;
+      if(ver < 31)
+      {
+        m_background_sync_data = background_sync_data_t{};
+        return;
+      }
+      a & m_background_sync_data;
     }
 
     BEGIN_SERIALIZE_OBJECT()
       MAGIC_FIELD("monero wallet cache")
-      VERSION_FIELD(1)
+      VERSION_FIELD(2)
       FIELD(m_blockchain)
       FIELD(m_transfers)
       FIELD(m_account_public_address)
@@ -1306,6 +1381,12 @@ private:
         return true;
       }
       FIELD(m_has_ever_refreshed_from_node)
+      if (version < 2)
+      {
+        m_background_sync_data = background_sync_data_t{};
+        return true;
+      }
+      FIELD(m_background_sync_data)
     END_SERIALIZE()
 
     /*!
@@ -1321,6 +1402,8 @@ private:
      * \return                Whether path is valid format
      */
     static bool wallet_valid_path_format(const std::string& file_path);
+    static std::string make_background_wallet_file_name(const std::string &wallet_file);
+    static std::string make_background_keys_file_name(const std::string &wallet_file);
     static bool parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id);
     static bool parse_short_payment_id(const std::string& payment_id_str, crypto::hash8& payment_id);
     static bool parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id);
@@ -1367,6 +1450,9 @@ private:
     void ignore_outputs_below(uint64_t value) { m_ignore_outputs_below = value; }
     bool track_uses() const { return m_track_uses; }
     void track_uses(bool value) { m_track_uses = value; }
+    BackgroundSyncType background_sync_type() const { return m_background_sync_type; }
+    void setup_background_sync(BackgroundSyncType background_sync_type, const epee::wipeable_string &wallet_password, const boost::optional<epee::wipeable_string> &background_cache_password);
+    bool is_background_syncing() const { return m_background_syncing; }
     bool show_wallet_name_when_locked() const { return m_show_wallet_name_when_locked; }
     void show_wallet_name_when_locked(bool value) { m_show_wallet_name_when_locked = value; }
     BackgroundMiningSetupType setup_background_mining() const { return m_setup_background_mining; }
@@ -1641,6 +1727,9 @@ private:
     uint64_t get_bytes_sent() const;
     uint64_t get_bytes_received() const;
 
+    void start_background_sync();
+    void stop_background_sync(const epee::wipeable_string &wallet_password, const crypto::secret_key &spend_secret_key = crypto::null_skey);
+
     // MMS -------------------------------------------------------------------------------------------------
     mms::message_store& get_message_store() { return m_message_store; };
     const mms::message_store& get_message_store() const { return m_message_store; };
@@ -1673,6 +1762,9 @@ private:
      * \return                Whether it was successful.
      */
     bool store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only = false);
+    bool store_keys(const std::string& keys_file_name, const crypto::chacha_key& key, bool watch_only = false, bool background_keys_file = false);
+    boost::optional<wallet2::keys_file_data> get_keys_file_data(const crypto::chacha_key& key, bool watch_only = false, bool background_keys_file = false);
+    bool store_keys_file_data(const std::string& keys_file_name, wallet2::keys_file_data &keys_file_data, bool background_keys_file = false);
     /*!
      * \brief Load wallet keys information from wallet file.
      * \param keys_file_name Name of wallet file
@@ -1686,6 +1778,7 @@ private:
      */
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password);
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt);
+    void load_wallet_cache(const bool use_fs, const std::string& cache_buf = "");
     void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL, bool ignore_callbacks = false);
     bool should_skip_block(const cryptonote::block &b, uint64_t height) const;
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
@@ -1694,6 +1787,15 @@ private:
     void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const;
     bool clear();
     void clear_soft(bool keep_key_images=false);
+    /*
+     * clear_user_data clears data created by the user, which is mostly data
+     * that a view key cannot identify on chain. This function was initially
+     * added to ensure that a "background" wallet (a wallet that syncs with just
+     * a view key hot in memory) does not have any sensitive data loaded that it
+     * does not need in order to sync. Future devs should take care to ensure
+     * that this function deletes data that is not useful for background syncing
+     */
+    void clear_user_data();
     void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>>& process_pool_txs);
     void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes);
     void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false);
@@ -1745,10 +1847,23 @@ private:
     bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
     crypto::chacha_key get_ringdb_key();
     void setup_keys(const epee::wipeable_string &password);
+    const crypto::chacha_key get_cache_key();
+    void verify_password_with_cached_key(const epee::wipeable_string &password);
+    void verify_password_with_cached_key(const crypto::chacha_key &key);
     size_t get_transfer_details(const crypto::key_image &ki) const;
     tx_entry_data get_tx_entries(const std::unordered_set<crypto::hash> &txids);
     void sort_scan_tx_entries(std::vector<process_tx_entry_t> &unsorted_tx_entries);
     void process_scan_txs(const tx_entry_data &txs_to_scan, const tx_entry_data &txs_to_reprocess, const std::unordered_set<crypto::hash> &tx_hashes_to_reprocess, detached_blockchain_data &dbd);
+    void write_background_sync_wallet(const epee::wipeable_string &wallet_password, const epee::wipeable_string &background_cache_password);
+    void process_background_cache_on_open();
+    void process_background_cache(const background_sync_data_t &background_sync_data, const hashchain &background_chain, uint64_t last_block_reward);
+    void reset_background_sync_data(background_sync_data_t &background_sync_data);
+    void store_background_cache(const crypto::chacha_key &custom_background_key, const bool do_reset_background_sync_data = true);
+    void store_background_keys(const crypto::chacha_key &custom_background_key);
+
+    bool lock_background_keys_file(const std::string &background_keys_file);
+    bool unlock_background_keys_file();
+    bool is_background_keys_file_locked() const;
 
     void register_devices();
     hw::device& lookup_device(const std::string & device_descriptor);
@@ -1860,6 +1975,8 @@ private:
     uint64_t m_ignore_outputs_above;
     uint64_t m_ignore_outputs_below;
     bool m_track_uses;
+    bool m_is_background_wallet;
+    BackgroundSyncType m_background_sync_type;
     bool m_show_wallet_name_when_locked;
     uint32_t m_inactivity_lock_timeout;
     BackgroundMiningSetupType m_setup_background_mining;
@@ -1887,6 +2004,7 @@ private:
 
     uint64_t m_last_block_reward;
     std::unique_ptr<tools::file_locker> m_keys_file_locker;
+    std::unique_ptr<tools::file_locker> m_background_keys_file_locker;
     
     mms::message_store m_message_store;
     bool m_original_keys_available;
@@ -1894,6 +2012,7 @@ private:
     crypto::secret_key m_original_view_secret_key;
 
     crypto::chacha_key m_cache_key;
+    boost::optional<crypto::chacha_key> m_custom_background_key = boost::none;
     std::shared_ptr<wallet_keys_unlocker> m_encrypt_keys_after_refresh;
 
     bool m_unattended;
@@ -1909,9 +2028,13 @@ private:
 
     static boost::mutex default_daemon_address_lock;
     static std::string default_daemon_address;
+
+    bool m_background_syncing;
+    bool m_processing_background_cache;
+    background_sync_data_t m_background_sync_data;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 30)
+BOOST_CLASS_VERSION(tools::wallet2, 31)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 12)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)
@@ -1927,6 +2050,8 @@ BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4)
 BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 1)
+BOOST_CLASS_VERSION(tools::wallet2::background_synced_tx_t, 0)
+BOOST_CLASS_VERSION(tools::wallet2::background_sync_data_t, 0)
 
 namespace boost
 {
@@ -2425,6 +2550,29 @@ namespace boost
         return;
       a & x.multisig_sigs;
     }
+
+    template <class Archive>
+    inline void serialize(Archive& a, tools::wallet2::background_synced_tx_t &x, const boost::serialization::version_type ver)
+    {
+      a & x.index_in_background_sync_data;
+      a & x.tx;
+      a & x.output_indices;
+      a & x.height;
+      a & x.block_timestamp;
+      a & x.double_spend_seen;
+    }
+
+    template <class Archive>
+    inline void serialize(Archive& a, tools::wallet2::background_sync_data_t &x, const boost::serialization::version_type ver)
+    {
+      a & x.first_refresh_done;
+      a & x.start_height;
+      a & x.txs;
+      a & x.wallet_refresh_from_block_height;
+      a & x.subaddress_lookahead_major;
+      a & x.subaddress_lookahead_minor;
+      a & x.wallet_refresh_type;
+    }
   }
 }
 

src/wallet/wallet_errors.h

@@ -63,6 +63,7 @@ namespace tools
     //       invalid_password
     //       invalid_priority
     //       invalid_multisig_seed
+    //       invalid_spend_key
     //       refresh_error *
     //         acc_outs_lookup_error
     //         block_parse_error
@@ -97,6 +98,9 @@ namespace tools
     //       wallet_files_doesnt_correspond
     //       scan_tx_error *
     //         wont_reprocess_recent_txs_via_untrusted_daemon
+    //       background_sync_error *
+    //         background_wallet_already_open
+    //         background_custom_password_same_as_wallet_password
     //
     // * - class with protected ctor
 
@@ -304,6 +308,16 @@ namespace tools
       std::string to_string() const { return wallet_logic_error::to_string(); }
     };
 
+    struct invalid_spend_key : public wallet_logic_error
+    {
+      explicit invalid_spend_key(std::string&& loc)
+        : wallet_logic_error(std::move(loc), "invalid spend key")
+      {
+      }
+
+      std::string to_string() const { return wallet_logic_error::to_string(); }
+    };
+
     //----------------------------------------------------------------------------------------------------
     struct invalid_pregenerated_random : public wallet_logic_error
     {
@@ -948,6 +962,31 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct background_sync_error : public wallet_logic_error
+    {
+    protected:
+      explicit background_sync_error(std::string&& loc, const std::string& message)
+        : wallet_logic_error(std::move(loc), message)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
+    struct background_wallet_already_open : public background_sync_error
+    {
+      explicit background_wallet_already_open(std::string&& loc, const std::string& background_wallet_file)
+        : background_sync_error(std::move(loc), "background wallet " + background_wallet_file + " is already opened by another wallet program")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
+    struct background_custom_password_same_as_wallet_password : public background_sync_error
+    {
+      explicit background_custom_password_same_as_wallet_password(std::string&& loc)
+        : background_sync_error(std::move(loc), "custom background password must be different than wallet password")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
 
 #if !defined(_MSC_VER)
 

src/wallet/wallet_rpc_server.cpp

@@ -73,6 +73,54 @@ using namespace epee;
     } \
   } while(0)
 
+#define CHECK_IF_BACKGROUND_SYNCING() \
+  do \
+  { \
+    if (!m_wallet) { return not_open(er); } \
+    if (m_wallet->is_background_wallet()) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_IS_BACKGROUND_WALLET; \
+      er.message = "This command is disabled for background wallets."; \
+      return false; \
+    } \
+    if (m_wallet->is_background_syncing()) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_IS_BACKGROUND_SYNCING; \
+      er.message = "This command is disabled while background syncing. Stop background syncing to use this command."; \
+      return false; \
+    } \
+  } while(0)
+
+#define PRE_VALIDATE_BACKGROUND_SYNC() \
+  do \
+  { \
+    if (!m_wallet) { return not_open(er); } \
+    if (m_restricted) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_DENIED; \
+      er.message = "Command unavailable in restricted mode."; \
+      return false; \
+    } \
+    if (m_wallet->key_on_device()) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; \
+      er.message = "Command not supported by HW wallet"; \
+      return false; \
+    } \
+    if (m_wallet->get_multisig_status().multisig_is_active) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; \
+      er.message = "Multisig wallet cannot enable background sync"; \
+      return false; \
+    } \
+    if (m_wallet->watch_only()) \
+    { \
+      er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; \
+      er.message = "Watch-only wallet cannot enable background sync"; \
+      return false; \
+    } \
+  } while (0)
+
 namespace
 {
   const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"};
@@ -291,6 +339,9 @@ namespace tools
   {
     if (!m_wallet)
       return;
+    // Background mining can be toggled from the main wallet
+    if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing())
+      return;
 
     tools::wallet2::BackgroundMiningSetupType setup = m_wallet->setup_background_mining();
     if (setup == tools::wallet2::BackgroundMiningNo)
@@ -581,6 +632,7 @@ namespace tools
   bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       if (req.count < 1 || req.count > 65536) {
@@ -618,6 +670,7 @@ namespace tools
   bool wallet_rpc_server::on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->set_subaddress_label(req.index, req.label);
@@ -680,6 +733,7 @@ namespace tools
   bool wallet_rpc_server::on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->add_subaddress_account(req.label);
@@ -697,6 +751,7 @@ namespace tools
   bool wallet_rpc_server::on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->set_subaddress_label({req.account_index, 0}, req.label);
@@ -712,6 +767,7 @@ namespace tools
   bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags();
     for (const std::pair<const std::string, std::string>& p : account_tags.first)
     {
@@ -731,6 +787,7 @@ namespace tools
   bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->set_account_tag(req.accounts, req.tag);
@@ -746,6 +803,7 @@ namespace tools
   bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->set_account_tag(req.accounts, "");
@@ -761,6 +819,7 @@ namespace tools
   bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->set_account_tag_description(req.tag, req.description);
@@ -791,6 +850,7 @@ namespace tools
   bool wallet_rpc_server::on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       if (req.key_image.empty())
@@ -819,6 +879,7 @@ namespace tools
   bool wallet_rpc_server::on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       if (req.key_image.empty())
@@ -847,6 +908,7 @@ namespace tools
   bool wallet_rpc_server::on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       if (req.key_image.empty())
@@ -874,6 +936,8 @@ namespace tools
   //------------------------------------------------------------------------------------------------------------------------------
   bool wallet_rpc_server::validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er)
   {
+    CHECK_IF_BACKGROUND_SYNCING();
+
     crypto::hash8 integrated_payment_id = crypto::null_hash8;
     std::string extra_nonce;
     for (auto it = destinations.begin(); it != destinations.end(); it++)
@@ -1203,6 +1267,7 @@ namespace tools
     }
 
     CHECK_MULTISIG_ENABLED();
+    CHECK_IF_BACKGROUND_SYNCING();
 
     cryptonote::blobdata blob;
     if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob))
@@ -1284,6 +1349,7 @@ namespace tools
       er.message = "command not supported by watch-only wallet";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
     if(req.unsigned_txset.empty() && req.multisig_txset.empty())
     {
       er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
@@ -1553,6 +1619,7 @@ namespace tools
     }
 
     CHECK_MULTISIG_ENABLED();
+    CHECK_IF_BACKGROUND_SYNCING();
 
     try
     {
@@ -2115,6 +2182,7 @@ namespace tools
             er.message = "The wallet is watch-only. Cannot retrieve seed.";
             return false;
           }
+          CHECK_IF_BACKGROUND_SYNCING();
           if (!m_wallet->is_deterministic())
           {
             er.code = WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC;
@@ -2143,6 +2211,7 @@ namespace tools
             er.message = "The wallet is watch-only. Cannot retrieve spend key.";
             return false;
           }
+          CHECK_IF_BACKGROUND_SYNCING();
           epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_spend_secret_key);
           res.key = std::string(key.data(), key.size());
       }
@@ -2164,6 +2233,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     try
     {
@@ -2177,6 +2247,79 @@ namespace tools
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_setup_background_sync(const wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx)
+  {
+    try
+    {
+      PRE_VALIDATE_BACKGROUND_SYNC();
+      const tools::wallet2::BackgroundSyncType background_sync_type = tools::wallet2::background_sync_type_from_str(req.background_sync_type);
+      boost::optional<epee::wipeable_string> background_cache_password = boost::none;
+      if (background_sync_type == tools::wallet2::BackgroundSyncCustomPassword)
+        background_cache_password = boost::optional<epee::wipeable_string>(req.background_cache_password);
+      m_wallet->setup_background_sync(background_sync_type, req.wallet_password, background_cache_password);
+    }
+    catch (...)
+    {
+      handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
+      return false;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_start_background_sync(const wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx)
+  {
+    try
+    {
+      PRE_VALIDATE_BACKGROUND_SYNC();
+      m_wallet->start_background_sync();
+    }
+    catch (...)
+    {
+      handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
+      return false;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_stop_background_sync(const wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx)
+  {
+    try
+    {
+      PRE_VALIDATE_BACKGROUND_SYNC();
+      crypto::secret_key spend_secret_key = crypto::null_skey;
+
+      // Load the spend key from seed if seed is provided
+      if (!req.seed.empty())
+      {
+        crypto::secret_key recovery_key;
+        std::string language;
+
+        if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, language))
+        {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = "Electrum-style word list failed verification";
+          return false;
+        }
+
+        if (!req.seed_offset.empty())
+          recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset);
+
+        // generate spend key
+        cryptonote::account_base account;
+        account.generate(recovery_key, true, false);
+        spend_secret_key = account.get_keys().m_spend_secret_key;
+      }
+
+      m_wallet->stop_background_sync(req.wallet_password, spend_secret_key);
+    }
+    catch (...)
+    {
+      handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
+      return false;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
@@ -2186,6 +2329,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     tools::wallet2::message_signature_type_t signature_type = tools::wallet2::sign_with_spend_key;
     if (req.signature_type == "spend" || req.signature_type == "")
@@ -2278,6 +2422,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     if (req.txids.size() != req.notes.size())
     {
@@ -2350,6 +2495,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     m_wallet->set_attribute(req.key, req.value);
 
@@ -2377,6 +2523,7 @@ namespace tools
   bool wallet_rpc_server::on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
 
     crypto::hash txid;
     if (!epee::string_tools::hex_to_pod(req.txid, txid))
@@ -2468,6 +2615,7 @@ namespace tools
   bool wallet_rpc_server::on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
 
     crypto::hash txid;
     if (!epee::string_tools::hex_to_pod(req.txid, txid))
@@ -2584,6 +2732,7 @@ namespace tools
   bool wallet_rpc_server::on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
 
     boost::optional<std::pair<uint32_t, uint64_t>> account_minreserve;
     if (!req.all)
@@ -2826,6 +2975,7 @@ namespace tools
       er.message = "command not supported by HW wallet";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     try
     {
@@ -2855,6 +3005,7 @@ namespace tools
       er.message = "command not supported by HW wallet";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     cryptonote::blobdata blob;
     if (!epee::string_tools::parse_hexstr_to_binbuff(req.outputs_data_hex, blob))
@@ -2880,6 +3031,7 @@ namespace tools
   bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all);
@@ -2916,6 +3068,7 @@ namespace tools
       er.message = "This command requires a trusted daemon.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       std::vector<std::pair<crypto::key_image, crypto::signature>> ski;
@@ -2984,6 +3137,7 @@ namespace tools
   bool wallet_rpc_server::on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
     if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
     const auto ab = m_wallet->get_address_book();
     if (req.entries.empty())
     {
@@ -3029,6 +3183,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     cryptonote::address_parse_info info;
     er.message = "";
@@ -3071,6 +3226,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     const auto ab = m_wallet->get_address_book();
     if (req.index >= ab.size())
@@ -3133,6 +3289,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     const auto ab = m_wallet->get_address_book();
     if (req.index >= ab.size())
@@ -3203,6 +3360,7 @@ namespace tools
           er.message = "Command unavailable in restricted mode.";
           return false;
       }
+      CHECK_IF_BACKGROUND_SYNCING();
 
       std::unordered_set<crypto::hash> txids;
       std::list<std::string>::const_iterator i = req.txids.begin();
@@ -3242,6 +3400,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
     try
     {
       m_wallet->rescan_spent();
@@ -3506,6 +3665,7 @@ namespace tools
       er.message = "Command unavailable in restricted mode.";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
     if (m_wallet->verify_password(req.old_password))
     {
       try
@@ -4039,6 +4199,7 @@ namespace tools
       er.message = "wallet is watch-only and cannot be made multisig";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     res.multisig_info = m_wallet->get_multisig_first_kex_msg();
     return true;
@@ -4066,6 +4227,7 @@ namespace tools
       er.message = "wallet is watch-only and cannot be made multisig";
       return false;
     }
+    CHECK_IF_BACKGROUND_SYNCING();
 
     try
     {

src/wallet/wallet_rpc_server.h

@@ -159,6 +159,9 @@ namespace tools
         MAP_JON_RPC_WE("set_log_categories", on_set_log_categories, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES)
         MAP_JON_RPC_WE("estimate_tx_size_and_weight", on_estimate_tx_size_and_weight, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT)
         MAP_JON_RPC_WE("get_version",        on_get_version,        wallet_rpc::COMMAND_RPC_GET_VERSION)
+        MAP_JON_RPC_WE("setup_background_sync", on_setup_background_sync, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC)
+        MAP_JON_RPC_WE("start_background_sync", on_start_background_sync, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC)
+        MAP_JON_RPC_WE("stop_background_sync", on_stop_background_sync, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -250,6 +253,9 @@ namespace tools
       bool on_set_log_categories(const wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::request& req, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_estimate_tx_size_and_weight(const wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::request& req, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
+      bool on_setup_background_sync(const wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
+      bool on_start_background_sync(const wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
+      bool on_stop_background_sync(const wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
 
       //json rpc v2
       bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);

src/wallet/wallet_rpc_server_commands_defs.h

@@ -2700,5 +2700,69 @@ namespace wallet_rpc
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
+  struct COMMAND_RPC_SETUP_BACKGROUND_SYNC
+  {
+    struct request_t
+    {
+      std::string background_sync_type;
+      std::string wallet_password;
+      std::string background_cache_password;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(background_sync_type)
+        KV_SERIALIZE(wallet_password)
+        KV_SERIALIZE_OPT(background_cache_password, (std::string)"")
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<request_t> request;
+
+    struct response_t
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<response_t> response;
+  };
+
+  struct COMMAND_RPC_START_BACKGROUND_SYNC
+  {
+    struct request_t
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<request_t> request;
+
+    struct response_t
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<response_t> response;
+  };
+
+  struct COMMAND_RPC_STOP_BACKGROUND_SYNC
+  {
+    struct request_t
+    {
+      std::string wallet_password;
+      std::string seed;
+      std::string seed_offset;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(wallet_password)
+        KV_SERIALIZE_OPT(seed, (std::string)"")
+        KV_SERIALIZE_OPT(seed_offset, (std::string)"")
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<request_t> request;
+
+    struct response_t
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<response_t> response;
+  };
 }
 }

src/wallet/wallet_rpc_server_error_codes.h

@@ -81,3 +81,5 @@
 #define WALLET_RPC_ERROR_CODE_DISABLED               -48
 #define WALLET_RPC_ERROR_CODE_PROXY_ALREADY_DEFINED  -49
 #define WALLET_RPC_ERROR_CODE_NONZERO_UNLOCK_TIME    -50
+#define WALLET_RPC_ERROR_CODE_IS_BACKGROUND_WALLET   -51
+#define WALLET_RPC_ERROR_CODE_IS_BACKGROUND_SYNCING  -52

tests/functional_tests/transfer.py

@@ -30,6 +30,7 @@
 
 from __future__ import print_function
 import json
+import util_resources
 import pprint
 from deepdiff import DeepDiff
 pp = pprint.PrettyPrinter(indent=2)
@@ -46,6 +47,17 @@ seeds = [
     'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
 ]
 
+def diff_transfers(actual_transfers, expected_transfers, ignore_order = True):
+    # The payments containers aren't ordered; re-scanning can lead to diff orders
+    diff = DeepDiff(actual_transfers, expected_transfers, ignore_order = ignore_order)
+    if diff != {}:
+        pp.pprint(diff)
+    assert diff == {}
+
+def diff_incoming_transfers(actual_transfers, expected_transfers):
+    # wallet2 m_transfers container is ordered and order should be the same across rescans
+    diff_transfers(actual_transfers, expected_transfers, ignore_order = False)
+
 class TransferTest():
     def run_test(self):
         self.reset()
@@ -64,6 +76,8 @@ class TransferTest():
         self.check_multiple_submissions()
         self.check_scan_tx()
         self.check_subtract_fee_from_outputs()
+        self.check_background_sync()
+        self.check_background_sync_reorg_recovery()
 
     def reset(self):
         print('Resetting blockchain')
@@ -875,12 +889,6 @@ class TransferTest():
 
         print('Testing scan_tx')
 
-        def diff_transfers(actual_transfers, expected_transfers):
-            diff = DeepDiff(actual_transfers, expected_transfers)
-            if diff != {}:
-                pp.pprint(diff)
-            assert diff == {}
-
         # set up sender_wallet
         sender_wallet = self.wallet[0]
         try: sender_wallet.close_wallet()
@@ -1162,5 +1170,385 @@ class TransferTest():
         except AssertionError:
             pass
 
+    def check_background_sync(self):
+        daemon = Daemon()
+
+        print('Testing background sync')
+
+        # Some helper functions
+        def stop_with_wrong_inputs(wallet, wallet_password, seed = ''):
+            invalid = False
+            try: wallet.stop_background_sync(wallet_password = wallet_password, seed = seed)
+            except: invalid = True
+            assert invalid
+
+        def open_with_wrong_password(wallet, filename, password):
+            invalid_password = False
+            try: wallet.open_wallet(filename, password = password)
+            except: invalid_password = True
+            assert invalid_password
+
+        def restore_wallet(wallet, seed, filename = '', password = ''):
+            wallet.close_wallet()
+            if filename != '':
+                util_resources.remove_wallet_files(filename)
+            wallet.restore_deterministic_wallet(seed = seed, filename = filename, password = password)
+            wallet.auto_refresh(enable = False)
+            assert wallet.get_transfers() == {}
+
+        def assert_correct_transfers(wallet, expected_transfers, expected_inc_transfers, expected_balance):
+            diff_transfers(wallet.get_transfers(), expected_transfers)
+            diff_incoming_transfers(wallet.incoming_transfers(transfer_type = 'all'), expected_inc_transfers)
+            assert wallet.get_balance().balance == expected_balance
+
+        # Set up sender_wallet. Prepare to sweep single output to receiver.
+        # We're testing a sweep because it makes sure background sync can
+        # properly pick up txs which do not have a change output back to sender.
+        sender_wallet = self.wallet[0]
+        try: sender_wallet.close_wallet()
+        except: pass
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0])
+        sender_wallet.auto_refresh(enable = False)
+        sender_wallet.refresh()
+        res = sender_wallet.incoming_transfers(transfer_type = 'available')
+        unlocked = [x for x in res.transfers if x.unlocked and x.amount > 0]
+        assert len(unlocked) > 0
+        ki = unlocked[0].key_image
+        amount = unlocked[0].amount
+        spent_txid = unlocked[0].tx_hash
+        sender_wallet.refresh()
+        res = sender_wallet.get_transfers()
+        out_len = 0 if 'out' not in res else len(res.out)
+        sender_starting_balance = sender_wallet.get_balance().balance
+
+        # Background sync type options
+        reuse_password = sender_wallet.background_sync_options.reuse_password
+        custom_password = sender_wallet.background_sync_options.custom_password
+
+        # set up receiver_wallet
+        receiver_wallet = self.wallet[1]
+        try: receiver_wallet.close_wallet()
+        except: pass
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1])
+        receiver_wallet.auto_refresh(enable = False)
+        receiver_wallet.refresh()
+        res = receiver_wallet.get_transfers()
+        in_len = 0 if 'in' not in res else len(res['in'])
+        receiver_starting_balance = receiver_wallet.get_balance().balance
+
+        # transfer from sender_wallet to receiver_wallet
+        dst = '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW'
+        res = sender_wallet.sweep_single(dst, key_image = ki)
+        assert len(res.tx_hash) == 32*2
+        txid = res.tx_hash
+        assert res.fee > 0
+        fee = res.fee
+        assert res.amount == amount - fee
+
+        expected_sender_balance = sender_starting_balance - amount
+        expected_receiver_balance = receiver_starting_balance + (amount - fee)
+
+        print('Checking background sync on outgoing wallet')
+        sender_wallet.setup_background_sync(background_sync_type = reuse_password)
+        sender_wallet.start_background_sync()
+        # Mine block to an uninvolved wallet
+        daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1)
+        # sender should still be able to scan the transfer normally because we
+        # spent an output that had a known key image
+        sender_wallet.refresh()
+        transfers = sender_wallet.get_transfers()
+        assert 'pending' not in transfers or len(transfers.pending) == 0
+        assert 'pool' not in transfers or len (transfers.pool) == 0
+        assert len(transfers.out) == out_len + 1
+        tx = [x for x in transfers.out if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount - fee
+        assert tx.fee == fee
+        assert len(tx.destinations) == 1
+        assert tx.destinations[0].amount == amount - fee
+        assert tx.destinations[0].address == dst
+        incoming_transfers = sender_wallet.incoming_transfers(transfer_type = 'all')
+        assert len([x for x in incoming_transfers.transfers if x.tx_hash == spent_txid and x.key_image == ki and x.spent]) == 1
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        # Restore and check background syncing outgoing wallet
+        restore_wallet(sender_wallet, seeds[0])
+        sender_wallet.setup_background_sync(background_sync_type = reuse_password)
+        sender_wallet.start_background_sync()
+        sender_wallet.refresh()
+        for i, out_tx in enumerate(transfers.out):
+            if 'destinations' in out_tx:
+                del transfers.out[i]['destinations'] # destinations are not expected after wallet restore
+        # sender's balance should be higher because can't detect spends while
+        # background sync enabled, only receives
+        background_bal = sender_wallet.get_balance().balance
+        assert background_bal > expected_sender_balance
+        background_transfers = sender_wallet.get_transfers()
+        assert 'out' not in background_transfers or len(background_transfers.out) == 0
+        assert 'in' in background_transfers and len(background_transfers['in']) > 0
+        background_incoming_transfers = sender_wallet.incoming_transfers(transfer_type = 'all')
+        assert len(background_incoming_transfers) == len(incoming_transfers)
+        assert len([x for x in background_incoming_transfers.transfers if x.spent or x.key_image != '']) == 0
+        assert len([x for x in background_incoming_transfers.transfers if x.tx_hash == spent_txid]) == 1
+
+        # Try to stop background sync with the wrong seed
+        stop_with_wrong_inputs(sender_wallet, wallet_password = '', seed = seeds[1])
+
+        # Stop background sync and check transfers update correctly
+        sender_wallet.stop_background_sync(wallet_password = '', seed = seeds[0])
+        assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Check stopping a wallet with wallet files saved to disk
+        for background_sync_type in [reuse_password, custom_password]:
+            restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+            background_cache_password = None if background_sync_type == reuse_password else 'background_password'
+            sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password)
+            sender_wallet.start_background_sync()
+            sender_wallet.refresh()
+            assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal)
+            stop_with_wrong_inputs(sender_wallet, 'wrong_password')
+            sender_wallet.stop_background_sync(wallet_password = 'test_password')
+            assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Close wallet while background syncing, then reopen
+        for background_sync_type in [reuse_password, custom_password]:
+            restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+            background_cache_password = None if background_sync_type == reuse_password else 'background_password'
+            sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password)
+            sender_wallet.start_background_sync()
+            sender_wallet.refresh()
+            assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal)
+            sender_wallet.close_wallet()
+            open_with_wrong_password(sender_wallet, 'test1', 'wrong_password')
+            sender_wallet.open_wallet('test1', password = 'test_password')
+            # It should reopen with spend key loaded and correctly scan all transfers
+            assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Close wallet while syncing normally, then reopen
+        for background_sync_type in [reuse_password, custom_password]:
+            restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+            background_cache_password = None if background_sync_type == reuse_password else 'background_password'
+            sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password)
+            sender_wallet.refresh()
+            assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+            sender_wallet.close_wallet()
+            open_with_wrong_password(sender_wallet, 'test1', 'wrong_password')
+            sender_wallet.open_wallet('test1', password = 'test_password')
+            assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Create background cache using custom password, then use it to sync, then reopen main wallet
+        for background_cache_password in ['background_password', '']:
+            restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+            assert not util_resources.file_exists('test1.background')
+            assert not util_resources.file_exists('test1.background.keys')
+            sender_wallet.setup_background_sync(background_sync_type = custom_password, wallet_password = 'test_password', background_cache_password = background_cache_password)
+            assert util_resources.file_exists('test1.background')
+            assert util_resources.file_exists('test1.background.keys')
+            sender_wallet.close_wallet()
+            open_with_wrong_password(sender_wallet, 'test1.background', 'test_password')
+            sender_wallet.open_wallet('test1.background', password = background_cache_password)
+            sender_wallet.refresh()
+            assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal)
+            sender_wallet.close_wallet()
+            sender_wallet.open_wallet('test1', password = 'test_password')
+            assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Check that main wallet keeps background cache encrypted with custom password in sync
+        restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+        sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = 'background_password')
+        sender_wallet.refresh()
+        assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+        sender_wallet.close_wallet()
+        sender_wallet.open_wallet('test1.background', password = 'background_password')
+        assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        # Try using wallet password as custom background password
+        restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+        assert not util_resources.file_exists('test1.background')
+        assert not util_resources.file_exists('test1.background.keys')
+        same_password = False
+        try: sender_wallet.setup_background_sync(background_sync_type = custom_password, wallet_password = 'test_password', background_cache_password = 'test_password')
+        except: same_password = True
+        assert same_password
+        assert not util_resources.file_exists('test1.background')
+        assert not util_resources.file_exists('test1.background.keys')
+
+        # Turn off background sync
+        for background_sync_type in [reuse_password, custom_password]:
+            restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password')
+            background_cache_password = None if background_sync_type == reuse_password else 'background_password'
+            sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password)
+            if background_sync_type == custom_password:
+                assert util_resources.file_exists('test1.background')
+                assert util_resources.file_exists('test1.background.keys')
+                sender_wallet.close_wallet()
+                assert util_resources.file_exists('test1.background')
+                assert util_resources.file_exists('test1.background.keys')
+            else:
+                assert not util_resources.file_exists('test1.background')
+                assert not util_resources.file_exists('test1.background.keys')
+                sender_wallet.close_wallet()
+                assert not util_resources.file_exists('test1.background')
+                assert not util_resources.file_exists('test1.background.keys')
+            sender_wallet.open_wallet('test1', password = 'test_password')
+            sender_wallet.setup_background_sync(background_sync_type = sender_wallet.background_sync_options.off, wallet_password = 'test_password')
+            assert not util_resources.file_exists('test1.background')
+            assert not util_resources.file_exists('test1.background.keys')
+            sender_wallet.close_wallet()
+            assert not util_resources.file_exists('test1.background')
+            assert not util_resources.file_exists('test1.background.keys')
+            sender_wallet.open_wallet('test1', password = 'test_password')
+
+        # Sanity check against outgoing wallet restored at height 0
+        sender_wallet.close_wallet()
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = 0)
+        sender_wallet.refresh()
+        assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance)
+
+        print('Checking background sync on incoming wallet')
+        receiver_wallet.setup_background_sync(background_sync_type = reuse_password)
+        receiver_wallet.start_background_sync()
+        receiver_wallet.refresh()
+        transfers = receiver_wallet.get_transfers()
+        assert 'pending' not in transfers or len(transfers.pending) == 0
+        assert 'pool' not in transfers or len (transfers.pool) == 0
+        assert len(transfers['in']) == in_len + 1
+        tx = [x for x in transfers['in'] if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount - fee
+        assert tx.fee == fee
+        incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all')
+        assert len([x for x in incoming_transfers.transfers if x.tx_hash == txid and x.key_image == '' and not x.spent]) == 1
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        # Restore and check background syncing incoming wallet
+        restore_wallet(receiver_wallet, seeds[1])
+        receiver_wallet.setup_background_sync(background_sync_type = reuse_password)
+        receiver_wallet.start_background_sync()
+        receiver_wallet.refresh()
+        if 'out' in transfers:
+            for i, out_tx in enumerate(transfers.out):
+                if 'destinations' in out_tx:
+                    del transfers.out[i]['destinations'] # destinations are not expected after wallet restore
+        background_bal = receiver_wallet.get_balance().balance
+        assert background_bal >= expected_receiver_balance
+        background_transfers = receiver_wallet.get_transfers()
+        assert 'out' not in background_transfers or len(background_transfers.out) == 0
+        assert 'in' in background_transfers and len(background_transfers['in']) > 0
+        background_incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all')
+        assert len(background_incoming_transfers) == len(incoming_transfers)
+        assert len([x for x in background_incoming_transfers.transfers if x.spent or x.key_image != '']) == 0
+        assert len([x for x in background_incoming_transfers.transfers if x.tx_hash == txid]) == 1
+
+        # Stop background sync and check transfers update correctly
+        receiver_wallet.stop_background_sync(wallet_password = '', seed = seeds[1])
+        diff_transfers(receiver_wallet.get_transfers(), transfers)
+        incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all')
+        assert len(background_incoming_transfers) == len(incoming_transfers)
+        assert len([x for x in incoming_transfers.transfers if x.tx_hash == txid and x.key_image != '' and not x.spent]) == 1
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        # Check a fresh incoming wallet with wallet files saved to disk and encrypted with password
+        restore_wallet(receiver_wallet, seeds[1], 'test2', 'test_password')
+        receiver_wallet.setup_background_sync(background_sync_type = reuse_password, wallet_password = 'test_password')
+        receiver_wallet.start_background_sync()
+        receiver_wallet.refresh()
+        assert_correct_transfers(receiver_wallet, background_transfers, background_incoming_transfers, background_bal)
+        stop_with_wrong_inputs(receiver_wallet, 'wrong_password')
+        receiver_wallet.stop_background_sync(wallet_password = 'test_password')
+        assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance)
+
+        # Close receiver's wallet while background sync is enabled then reopen
+        restore_wallet(receiver_wallet, seeds[1], 'test2', 'test_password')
+        receiver_wallet.setup_background_sync(background_sync_type = reuse_password, wallet_password = 'test_password')
+        receiver_wallet.start_background_sync()
+        receiver_wallet.refresh()
+        diff_transfers(receiver_wallet.get_transfers(), background_transfers)
+        diff_incoming_transfers(receiver_wallet.incoming_transfers(transfer_type = 'all'), background_incoming_transfers)
+        assert receiver_wallet.get_balance().balance == background_bal
+        receiver_wallet.close_wallet()
+        receiver_wallet.open_wallet('test2', password = 'test_password')
+        # It should reopen with spend key loaded and correctly scan all transfers
+        assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance)
+
+        # Sanity check against incoming wallet restored at height 0
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = 0)
+        receiver_wallet.refresh()
+        assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance)
+
+        # Clean up
+        util_resources.remove_wallet_files('test1')
+        util_resources.remove_wallet_files('test2')
+        for i in range(2):
+            self.wallet[i].close_wallet()
+            self.wallet[i].restore_deterministic_wallet(seed = seeds[i])
+
+    def check_background_sync_reorg_recovery(self):
+        daemon = Daemon()
+
+        print('Testing background sync reorg recovery')
+
+        # Disconnect daemon from peers
+        daemon.out_peers(0)
+
+        # Background sync type options
+        sender_wallet = self.wallet[0]
+        reuse_password = sender_wallet.background_sync_options.reuse_password
+        custom_password = sender_wallet.background_sync_options.custom_password
+
+        for background_sync_type in [reuse_password, custom_password]:
+            # Set up wallet saved to disk
+            sender_wallet.close_wallet()
+            util_resources.remove_wallet_files('test1')
+            sender_wallet.restore_deterministic_wallet(seed = seeds[0], filename = 'test1', password = '')
+            sender_wallet.auto_refresh(enable = False)
+            sender_wallet.refresh()
+            sender_starting_balance = sender_wallet.get_balance().balance
+
+            # Send tx and mine a block
+            amount = 1000000000000
+            assert sender_starting_balance > amount
+            dst = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': amount}
+            res = sender_wallet.transfer([dst])
+            assert len(res.tx_hash) == 32*2
+            txid = res.tx_hash
+
+            daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1)
+
+            # Make sure the wallet can see the tx
+            sender_wallet.refresh()
+            transfers = sender_wallet.get_transfers()
+            assert 'pool' not in transfers or len (transfers.pool) == 0
+            tx = [x for x in transfers.out if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert sender_wallet.get_balance().balance < (sender_starting_balance - amount)
+
+            # Pop the block while background syncing
+            background_cache_password = None if background_sync_type == reuse_password else 'background_password'
+            sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = '', background_cache_password = background_cache_password)
+            sender_wallet.start_background_sync()
+            daemon.pop_blocks(1)
+            daemon.flush_txpool()
+
+            daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1)
+
+            # Make sure the wallet can no longer see the tx
+            sender_wallet.refresh()
+            sender_wallet.stop_background_sync(wallet_password = '', seed = seeds[0])
+            transfers = sender_wallet.get_transfers()
+            no_tx = [x for x in transfers.out if x.txid == txid]
+            assert len(no_tx) == 0
+            assert sender_wallet.get_balance().balance == sender_starting_balance
+
+        # Clean up
+        daemon.out_peers(12)
+        util_resources.remove_wallet_files('test1')
+        self.wallet[0].close_wallet()
+        self.wallet[0].restore_deterministic_wallet(seed = seeds[0])
+
 if __name__ == '__main__':
     TransferTest().run_test()

tests/functional_tests/util_resources.py

@@ -37,6 +37,8 @@
 from __future__ import print_function
 import subprocess
 import psutil
+import os
+import errno
 
 def available_ram_gb():
     ram_bytes = psutil.virtual_memory().available
@@ -51,3 +53,26 @@ def get_time_pi_seconds(cores, app_dir='.'):
     miliseconds = int(decoded)
 
     return miliseconds / 1000.0
+
+def remove_file(name):
+    WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY']
+    assert WALLET_DIRECTORY != ''
+    try:
+        os.unlink(WALLET_DIRECTORY + '/' + name)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+def get_file_path(name):
+    WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY']
+    assert WALLET_DIRECTORY != ''
+    return WALLET_DIRECTORY + '/' + name
+
+def remove_wallet_files(name):
+    for suffix in ['', '.keys', '.background', '.background.keys', '.address.txt']:
+        remove_file(name + suffix)
+
+def file_exists(name):
+    WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY']
+    assert WALLET_DIRECTORY != ''
+    return os.path.isfile(WALLET_DIRECTORY + '/' + name)

tests/functional_tests/wallet.py

@@ -34,8 +34,7 @@
 
 from __future__ import print_function
 import sys
-import os
+import util_resources
-import errno
 
 from framework.wallet import Wallet
 from framework.daemon import Daemon
@@ -54,24 +53,6 @@ class WalletTest():
       self.change_password()
       self.store()
 
-    def remove_file(self, name):
-        WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY']
-        assert WALLET_DIRECTORY != ''
-        try:
-            os.unlink(WALLET_DIRECTORY + '/' + name)
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                raise
-
-    def remove_wallet_files(self, name):
-        for suffix in ['', '.keys']:
-            self.remove_file(name + suffix)
-
-    def file_exists(self, name):
-        WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY']
-        assert WALLET_DIRECTORY != ''
-        return os.path.isfile(WALLET_DIRECTORY + '/' + name)
-
     def reset(self):
         print('Resetting blockchain')
         daemon = Daemon()
@@ -333,7 +314,7 @@ class WalletTest():
         try: wallet.close_wallet()
         except: pass
 
-        self.remove_wallet_files('test1')
+        util_resources.remove_wallet_files('test1')
 
         seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
         res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1')
@@ -359,7 +340,7 @@ class WalletTest():
 
         wallet.close_wallet()
 
-        self.remove_wallet_files('test1')
+        util_resources.remove_wallet_files('test1')
 
     def store(self):
         print('Testing store')
@@ -369,22 +350,26 @@ class WalletTest():
         try: wallet.close_wallet()
         except: pass
 
-        self.remove_wallet_files('test1')
+        util_resources.remove_wallet_files('test1')
 
         seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
         res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1')
         assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
         assert res.seed == seed
 
-        self.remove_file('test1')
+        util_resources.remove_file('test1')
-        assert self.file_exists('test1.keys')
+        assert util_resources.file_exists('test1.keys')
-        assert not self.file_exists('test1')
+        assert not util_resources.file_exists('test1')
         wallet.store()
-        assert self.file_exists('test1.keys')
+        assert util_resources.file_exists('test1.keys')
-        assert self.file_exists('test1')
+        assert util_resources.file_exists('test1')
 
         wallet.close_wallet()
-        self.remove_wallet_files('test1')
+
+        wallet.open_wallet(filename = 'test1', password = '')
+        wallet.close_wallet()
+
+        util_resources.remove_wallet_files('test1')
 
 
 if __name__ == '__main__':

tests/unit_tests/wipeable_string.cpp

@@ -211,3 +211,15 @@ TEST(wipeable_string, to_hex)
   ASSERT_TRUE(epee::to_hex::wipeable_string(epee::span<const uint8_t>((const uint8_t*)"", 0)) == epee::wipeable_string(""));
   ASSERT_TRUE(epee::to_hex::wipeable_string(epee::span<const uint8_t>((const uint8_t*)"abc", 3)) == epee::wipeable_string("616263"));
 }
+
+TEST(wipeable_string, to_string)
+{
+  // Converting a wipeable_string to a string defeats the purpose of wipeable_string,
+  // but nice to know this works
+  std::string str;
+  {
+    epee::wipeable_string wipeable_str("foo");
+    str = std::string(wipeable_str.data(), wipeable_str.size());
+  }
+  ASSERT_TRUE(str == std::string("foo"));
+}

utils/python-rpc/framework/wallet.py

@@ -1139,3 +1139,45 @@ class Wallet(object):
             'id': '0'
         }
         return self.rpc.send_json_rpc_request(frozen)
+
+    class BackgroundSyncOptions(object):
+        def __init__(self):
+            self.off = 'off'
+            self.reuse_password = 'reuse-wallet-password'
+            self.custom_password = 'custom-background-password'
+    background_sync_options = BackgroundSyncOptions()
+
+    def setup_background_sync(self, background_sync_type = background_sync_options.off, wallet_password = '', background_cache_password = ''):
+        setup_background_sync = {
+            'method': 'setup_background_sync',
+            'jsonrpc': '2.0',
+            'params' : {
+                'background_sync_type': background_sync_type,
+                'wallet_password': wallet_password,
+                'background_cache_password': background_cache_password,
+            },
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(setup_background_sync)
+
+    def start_background_sync(self):
+        start_background_sync = {
+            'method': 'start_background_sync',
+            'jsonrpc': '2.0',
+            'params' : {},
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(start_background_sync)
+
+    def stop_background_sync(self, wallet_password = '', seed = '', seed_offset = ''):
+        stop_background_sync = {
+            'method': 'stop_background_sync',
+            'jsonrpc': '2.0',
+            'params' : {
+                'wallet_password': wallet_password,
+                'seed': seed,
+                'seed_offset': seed_offset,
+            },
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(stop_background_sync)