Merge pull request #8513

959a3e6 wallet2: ensure imported outputs subaddresses are created (moneromooo-monero)
a098504 wallet2: better test on whether to allow output import (moneromooo-monero)
c5579ac allow exporting outputs in chunks (moneromooo-monero)
1e912ec wallet2: fixes for export/import output flow (j-berman)
692f1d4 wallet2: do not assume imported outputs must be non empty (moneromooo-monero)
67b6d6a wallet2: prevent importing outputs in a hot wallet (moneromooo-monero)
d9fc666 wallet2: fix missing subaddress indices in 'light' exported outputs (moneromooo-monero)

src/device_trezor/device_trezor.cpp

@@ -511,7 +511,7 @@ namespace trezor {
                                 tools::wallet2::signed_tx_set & signed_tx,
                                 hw::tx_aux_data & aux_data)
     {
-      CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset");
+      CHECK_AND_ASSERT_THROW_MES(std::get<0>(unsigned_tx.transfers) == 0, "Unsuported non zero offset");
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
@@ -522,7 +522,7 @@ namespace trezor {
       const size_t num_tx = unsigned_tx.txes.size();
       m_num_transations_to_sign = num_tx;
       signed_tx.key_images.clear();
-      signed_tx.key_images.resize(unsigned_tx.transfers.second.size());
+      signed_tx.key_images.resize(std::get<2>(unsigned_tx.transfers).size());
 
       for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) {
         std::shared_ptr<protocol::tx::Signer> signer;
@@ -566,8 +566,8 @@ namespace trezor {
         cpend.key_images = key_images;
 
         // KI sync
-        for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){
+        for(size_t cidx=0, trans_max=std::get<2>(unsigned_tx.transfers).size(); cidx < trans_max; ++cidx){
-          signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image;
+          signed_tx.key_images[cidx] = std::get<2>(unsigned_tx.transfers)[cidx].m_key_image;
         }
 
         size_t num_sources = cdata.tx_data.sources.size();
@@ -579,9 +579,9 @@ namespace trezor {
           CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped");
 
           size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped];
-          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset");
+          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= std::get<0>(unsigned_tx.transfers), "Invalid offset");
 
-          idx_map_src -= unsigned_tx.transfers.first;
+          idx_map_src -= std::get<0>(unsigned_tx.transfers);
           CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index");
 
           const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]);

src/device_trezor/trezor/protocol.hpp

@@ -230,8 +230,8 @@ namespace tx {
     }
 
     const tools::wallet2::transfer_details & get_transfer(size_t idx) const {
-      CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index");
+      CHECK_AND_ASSERT_THROW_MES(idx < std::get<2>(m_unsigned_tx->transfers).size() + std::get<0>(m_unsigned_tx->transfers) && idx >= std::get<0>(m_unsigned_tx->transfers), "Invalid transfer index");
-      return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first];
+      return std::get<2>(m_unsigned_tx->transfers)[idx - std::get<0>(m_unsigned_tx->transfers)];
     }
 
     const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const {

src/serialization/tuple.h

@@ -0,0 +1,169 @@
+// Copyright (c) 2014-2020, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+#include <memory>
+#include "serialization.h"
+
+namespace serialization
+{
+  namespace detail
+  {
+    template <typename Archive, class T>
+    bool serialize_tuple_element(Archive& ar, T& e)
+    {
+      return ::do_serialize(ar, e);
+    }
+
+    template <typename Archive>
+    bool serialize_tuple_element(Archive& ar, uint64_t& e)
+    {
+      ar.serialize_varint(e);
+      return true;
+    }
+  }
+}
+
+template <template <bool> class Archive, class E0, class E1, class E2>
+inline bool do_serialize(Archive<false>& ar, std::tuple<E0,E1,E2>& p)
+{
+  size_t cnt;
+  ar.begin_array(cnt);
+  if (!ar.good())
+    return false;
+  if (cnt != 3)
+    return false;
+
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<0>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<1>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<2>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+
+  ar.end_array();
+  return true;
+}
+
+template <template <bool> class Archive, class E0, class E1, class E2>
+inline bool do_serialize(Archive<true>& ar, std::tuple<E0,E1,E2>& p)
+{
+  ar.begin_array(3);
+  if (!ar.good())
+    return false;
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<0>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<1>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<2>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.end_array();
+  return true;
+}
+
+template <template <bool> class Archive, class E0, class E1, class E2, class E3>
+inline bool do_serialize(Archive<false>& ar, std::tuple<E0,E1,E2,E3>& p)
+{
+  size_t cnt;
+  ar.begin_array(cnt);
+  if (!ar.good())
+    return false;
+  if (cnt != 4)
+    return false;
+
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<0>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<1>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<2>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if (!::serialization::detail::serialize_tuple_element(ar, std::get<3>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+
+  ar.end_array();
+  return true;
+}
+
+template <template <bool> class Archive, class E0, class E1, class E2, class E3>
+inline bool do_serialize(Archive<true>& ar, std::tuple<E0,E1,E2,E3>& p)
+{
+  ar.begin_array(4);
+  if (!ar.good())
+    return false;
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<0>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<1>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<2>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.delimit_array();
+  if(!::serialization::detail::serialize_tuple_element(ar, std::get<3>(p)))
+    return false;
+  if (!ar.good())
+    return false;
+  ar.end_array();
+  return true;
+}
+

src/simplewallet/simplewallet.cpp

@@ -7912,8 +7912,10 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes,
 bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs)
 {
   std::string extra_message;
-  if (!txs.transfers.second.empty())
+  if (!std::get<2>(txs.new_transfers).empty())
-    extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str();
+    extra_message = (boost::format("%u outputs to import. ") % (unsigned)std::get<2>(txs.new_transfers).size()).str();
+  else if (!std::get<2>(txs.transfers).empty())
+    extra_message = (boost::format("%u outputs to import. ") % (unsigned)std::get<2>(txs.transfers).size()).str();
   return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message);
 }
 //----------------------------------------------------------------------------------------------------

src/wallet/api/wallet.cpp

@@ -1146,8 +1146,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file
   
   // Check tx data and construct confirmation message
   std::string extra_message;
-  if (!transaction->m_unsigned_tx_set.transfers.second.empty())
+  if (!std::get<2>(transaction->m_unsigned_tx_set.transfers).empty())
-    extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.second.size()).str();
+    extra_message = (boost::format("%u outputs to import. ") % (unsigned)std::get<2>(transaction->m_unsigned_tx_set.transfers).size()).str();
   transaction->checkLoadedTx([&transaction](){return transaction->m_unsigned_tx_set.txes.size();}, [&transaction](size_t n)->const tools::wallet2::tx_construction_data&{return transaction->m_unsigned_tx_set.txes[n];}, extra_message);
   setStatus(transaction->status(), transaction->errorString());
     

src/wallet/wallet2.cpp

@@ -1218,7 +1218,8 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std
   m_export_format(ExportFormat::Binary),
   m_load_deprecated_formats(false),
   m_credits_target(0),
-  m_enable_multisig(false)
+  m_enable_multisig(false),
+  m_has_ever_refreshed_from_node(false)
 {
   set_rpc_client_secret_key(rct::rct2sk(rct::skGen()));
 }
@@ -3539,6 +3540,8 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo
           throw std::runtime_error("proxy exception in refresh thread");
       }
 
+      m_has_ever_refreshed_from_node = true;
+
       if(!first && blocks_start_height == next_blocks_start_height)
       {
         m_node_rpc_proxy.set_height(m_blockchain.size());
@@ -6608,9 +6611,9 @@ bool wallet2::sign_tx(const std::string &unsigned_filename, const std::string &s
 //----------------------------------------------------------------------------------------------------
 bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &txs, signed_tx_set &signed_txes)
 {
-  if (!exported_txs.new_transfers.second.empty())
+  if (!std::get<2>(exported_txs.new_transfers).empty())
     import_outputs(exported_txs.new_transfers);
-  else
+  else if (!std::get<2>(exported_txs.transfers).empty())
     import_outputs(exported_txs.transfers);
 
   // sign the transactions
@@ -10804,7 +10807,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_
   {
     txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device()));
   }
-  txs.transfers = std::make_pair(0, m_transfers);
+  txs.transfers = std::make_tuple(0, m_transfers.size(), m_transfers);
 
   auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev);
   CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
@@ -13107,18 +13110,29 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect
   m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx);
 }
 //----------------------------------------------------------------------------------------------------
-std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wallet2::export_outputs(bool all) const
+std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wallet2::export_outputs(bool all, uint32_t start, uint32_t count) const
 {
   PERF_TIMER(export_outputs);
   std::vector<tools::wallet2::exported_transfer_details> outs;
 
+  // invalid cases
+  THROW_WALLET_EXCEPTION_IF(count == 0, error::wallet_internal_error, "Nothing requested");
+  THROW_WALLET_EXCEPTION_IF(!all && start > 0, error::wallet_internal_error, "Incremental mode is incompatible with non-zero start");
+
+  // valid cases:
+  // all: all outputs, subject to start/count
+  // !all: incremental, subject to count
+  // for convenience, start/count are allowed to go past the valid range, then nothing is returned
+
   size_t offset = 0;
   if (!all)
     while (offset < m_transfers.size() && (m_transfers[offset].m_key_image_known && !m_transfers[offset].m_key_image_request))
       ++offset;
+  else
+    offset = start;
 
   outs.reserve(m_transfers.size() - offset);
-  for (size_t n = offset; n < m_transfers.size(); ++n)
+  for (size_t n = offset; n < m_transfers.size() && n - offset < count; ++n)
   {
     const transfer_details &td = m_transfers[n];
 
@@ -13136,20 +13150,22 @@ std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wall
     etd.m_flags.m_key_image_partial = td.m_key_image_partial;
     etd.m_amount = td.m_amount;
     etd.m_additional_tx_keys = get_additional_tx_pub_keys_from_extra(td.m_tx);
+    etd.m_subaddr_index_major = td.m_subaddr_index.major;
+    etd.m_subaddr_index_minor = td.m_subaddr_index.minor;
 
     outs.push_back(etd);
   }
 
-  return std::make_pair(offset, outs);
+  return std::make_tuple(offset, m_transfers.size(), outs);
 }
 //----------------------------------------------------------------------------------------------------
-std::string wallet2::export_outputs_to_str(bool all) const
+std::string wallet2::export_outputs_to_str(bool all, uint32_t start, uint32_t count) const
 {
   PERF_TIMER(export_outputs_to_str);
 
   std::stringstream oss;
   binary_archive<true> ar(oss);
-  auto outputs = export_outputs(all);
+  auto outputs = export_outputs(all, start, count);
   THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, outputs), error::wallet_internal_error, "Failed to serialize output data");
 
   std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC));
@@ -13162,21 +13178,35 @@ std::string wallet2::export_outputs_to_str(bool all) const
   return magic + ciphertext;
 }
 //----------------------------------------------------------------------------------------------------
-size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs)
+size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs)
 {
   PERF_TIMER(import_outputs);
 
-  THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error,
+  THROW_WALLET_EXCEPTION_IF(m_has_ever_refreshed_from_node, error::wallet_internal_error,
+      "Hot wallets cannot import outputs");
+
+  // we can now import piecemeal
+  const size_t offset = std::get<0>(outputs);
+  const size_t num_outputs = std::get<1>(outputs);
+  const std::vector<tools::wallet2::transfer_details> &output_array = std::get<2>(outputs);
+
+  THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error,
       "Imported outputs omit more outputs that we know of");
 
-  const size_t offset = outputs.first;
+  THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+  THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+
   const size_t original_size = m_transfers.size();
-  m_transfers.resize(offset + outputs.second.size());
+  if (offset + output_array.size() > m_transfers.size())
-  for (size_t i = 0; i < offset; ++i)
+    m_transfers.resize(offset + output_array.size());
-    m_transfers[i].m_key_image_request = false;
+  else if (num_outputs < m_transfers.size())
-  for (size_t i = 0; i < outputs.second.size(); ++i)
+    m_transfers.resize(num_outputs);
+
+  for (size_t i = 0; i < output_array.size(); ++i)
   {
-    transfer_details td = outputs.second[i];
+    transfer_details td = output_array[i];
 
     // skip those we've already imported, or which have different data
     if (i + offset < original_size)
@@ -13210,6 +13240,8 @@ process:
     THROW_WALLET_EXCEPTION_IF(td.m_internal_output_index >= td.m_tx.vout.size(),
         error::wallet_internal_error, "Internal index is out of range");
     crypto::public_key out_key = td.get_public_key();
+    if (should_expand(td.m_subaddr_index))
+      create_one_off_subaddress(td.m_subaddr_index);
     bool r = cryptonote::generate_key_image_helper(m_account.get_keys(), m_subaddresses, out_key, tx_pub_key, additional_tx_pub_keys, td.m_internal_output_index, in_ephemeral, td.m_key_image, m_account.get_device());
     THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image");
     if (should_expand(td.m_subaddr_index))
@@ -13228,24 +13260,38 @@ process:
   return m_transfers.size();
 }
 //----------------------------------------------------------------------------------------------------
-size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs)
+size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs)
 {
   PERF_TIMER(import_outputs);
 
-  THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error,
+  THROW_WALLET_EXCEPTION_IF(m_has_ever_refreshed_from_node, error::wallet_internal_error,
+      "Hot wallets cannot import outputs");
+
+  // we can now import piecemeal
+  const size_t offset = std::get<0>(outputs);
+  const size_t num_outputs = std::get<1>(outputs);
+  const std::vector<tools::wallet2::exported_transfer_details> &output_array = std::get<2>(outputs);
+
+  THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error,
       "Imported outputs omit more outputs that we know of. Try using export_outputs all.");
 
-  const size_t offset = outputs.first;
+  THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+  THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+
   const size_t original_size = m_transfers.size();
-  m_transfers.resize(offset + outputs.second.size());
+  if (offset + output_array.size() > m_transfers.size())
-  for (size_t i = 0; i < offset; ++i)
+    m_transfers.resize(offset + output_array.size());
-    m_transfers[i].m_key_image_request = false;
+  else if (num_outputs < m_transfers.size())
-  for (size_t i = 0; i < outputs.second.size(); ++i)
+    m_transfers.resize(num_outputs);
+
+  for (size_t i = 0; i < output_array.size(); ++i)
   {
-    exported_transfer_details etd = outputs.second[i];
+    exported_transfer_details etd = output_array[i];
     transfer_details &td = m_transfers[i + offset];
 
-    // setup td with "cheao" loaded data
+    // setup td with "cheap" loaded data
     td.m_block_height = 0;
     td.m_txid = crypto::null_hash;
     td.m_global_output_index = etd.m_global_output_index;
@@ -13258,6 +13304,8 @@ size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wall
     td.m_key_image_known = etd.m_flags.m_key_image_known;
     td.m_key_image_request = etd.m_flags.m_key_image_request;
     td.m_key_image_partial = false;
+    td.m_subaddr_index.major = etd.m_subaddr_index_major;
+    td.m_subaddr_index.minor = etd.m_subaddr_index_minor;
 
     // skip those we've already imported, or which have different data
     if (i + offset < original_size)
@@ -13298,6 +13346,8 @@ size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wall
     const crypto::public_key &tx_pub_key = etd.m_tx_pubkey;
     const std::vector<crypto::public_key> &additional_tx_pub_keys = etd.m_additional_tx_keys;
     const crypto::public_key& out_key = etd.m_pubkey;
+    if (should_expand(td.m_subaddr_index))
+      create_one_off_subaddress(td.m_subaddr_index);
     bool r = cryptonote::generate_key_image_helper(m_account.get_keys(), m_subaddresses, out_key, tx_pub_key, additional_tx_pub_keys, td.m_internal_output_index, in_ephemeral, td.m_key_image, m_account.get_device());
     THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image");
     if (should_expand(td.m_subaddr_index))
@@ -13354,7 +13404,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
   {
     std::string body(data, headerlen);
 
-    std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> new_outputs;
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> new_outputs;
     try
     {
       binary_archive<false> ar{epee::strspan<std::uint8_t>(body)};
@@ -13364,9 +13414,9 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
     }
     catch (...) {}
     if (!loaded)
-      new_outputs.second.clear();
+      std::get<2>(new_outputs).clear();
 
-    std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
     if (!loaded) try
     {
       binary_archive<false> ar{epee::strspan<std::uint8_t>(body)};
@@ -13391,11 +13441,12 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
 
     if (!loaded)
     {
-      outputs.first = 0;
+      std::get<0>(outputs) = 0;
-      outputs.second = {};
+      std::get<1>(outputs) = 0;
+      std::get<2>(outputs) = {};
     }
 
-    imported_outputs = new_outputs.second.empty() ? import_outputs(outputs) : import_outputs(new_outputs);
+    imported_outputs = !std::get<2>(new_outputs).empty() ? import_outputs(new_outputs) : !std::get<2>(outputs).empty() ? import_outputs(outputs) : 0;
   }
   catch (const std::exception &e)
   {

src/wallet/wallet2.h

@@ -63,6 +63,7 @@
 #include "serialization/crypto.h"
 #include "serialization/string.h"
 #include "serialization/pair.h"
+#include "serialization/tuple.h"
 #include "serialization/containers.h"
 
 #include "wallet_errors.h"
@@ -401,9 +402,13 @@ private:
       } m_flags;
       uint64_t m_amount;
       std::vector<crypto::public_key> m_additional_tx_keys;
+      uint32_t m_subaddr_index_major;
+      uint32_t m_subaddr_index_minor;
 
       BEGIN_SERIALIZE_OBJECT()
-        VERSION_FIELD(0)
+        VERSION_FIELD(1)
+        if (version < 1)
+          return false;
         FIELD(m_pubkey)
         VARINT_FIELD(m_internal_output_index)
         VARINT_FIELD(m_global_output_index)
@@ -411,6 +416,8 @@ private:
         FIELD(m_flags.flags)
         VARINT_FIELD(m_amount)
         FIELD(m_additional_tx_keys)
+        VARINT_FIELD(m_subaddr_index_major)
+        VARINT_FIELD(m_subaddr_index_minor)
       END_SERIALIZE()
     };
 
@@ -667,16 +674,32 @@ private:
     struct unsigned_tx_set
     {
       std::vector<tx_construction_data> txes;
-      std::pair<size_t, wallet2::transfer_container> transfers;
+      std::tuple<uint64_t, uint64_t, wallet2::transfer_container> transfers;
-      std::pair<size_t, std::vector<wallet2::exported_transfer_details>> new_transfers;
+      std::tuple<uint64_t, uint64_t, std::vector<wallet2::exported_transfer_details>> new_transfers;
 
       BEGIN_SERIALIZE_OBJECT()
-        VERSION_FIELD(1)
+        VERSION_FIELD(2)
         FIELD(txes)
-        if (version >= 1)
+        if (version == 0)
+        {
+          std::pair<size_t, wallet2::transfer_container> v0_transfers;
+          FIELD(v0_transfers);
+          std::get<0>(transfers) = std::get<0>(v0_transfers);
+          std::get<1>(transfers) = std::get<0>(v0_transfers) + std::get<1>(v0_transfers).size();
+          std::get<2>(transfers) = std::get<1>(v0_transfers);
+          return true;
+        }
+        if (version == 1)
+        {
+          std::pair<size_t, std::vector<wallet2::exported_transfer_details>> v1_transfers;
+          FIELD(v1_transfers);
+          std::get<0>(new_transfers) = std::get<0>(v1_transfers);
+          std::get<1>(new_transfers) = std::get<0>(v1_transfers) + std::get<1>(v1_transfers).size();
+          std::get<2>(new_transfers) = std::get<1>(v1_transfers);
+          return true;
+        }
+
         FIELD(new_transfers)
-        else
-          FIELD(transfers)
       END_SERIALIZE()
     };
 
@@ -1206,11 +1229,17 @@ private:
       if(ver < 29)
         return;
       a & m_rpc_client_secret_key;
+      if(ver < 30)
+      {
+        m_has_ever_refreshed_from_node = false;
+        return;
+      }
+      a & m_has_ever_refreshed_from_node;
     }
 
     BEGIN_SERIALIZE_OBJECT()
       MAGIC_FIELD("monero wallet cache")
-      VERSION_FIELD(0)
+      VERSION_FIELD(1)
       FIELD(m_blockchain)
       FIELD(m_transfers)
       FIELD(m_account_public_address)
@@ -1236,6 +1265,12 @@ private:
       FIELD(m_device_last_key_image_sync)
       FIELD(m_cold_key_images)
       FIELD(m_rpc_client_secret_key)
+      if (version < 1)
+      {
+        m_has_ever_refreshed_from_node = false;
+        return true;
+      }
+      FIELD(m_has_ever_refreshed_from_node)
     END_SERIALIZE()
 
     /*!
@@ -1447,10 +1482,10 @@ private:
     bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const;
 
     // Import/Export wallet data
-    std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> export_outputs(bool all = false) const;
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> export_outputs(bool all = false, uint32_t start = 0, uint32_t count = 0xffffffff) const;
-    std::string export_outputs_to_str(bool all = false) const;
+    std::string export_outputs_to_str(bool all = false, uint32_t start = 0, uint32_t count = 0xffffffff) const;
-    size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs);
+    size_t import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs);
-    size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs);
+    size_t import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs);
     size_t import_outputs_from_str(const std::string &outputs_st);
     payment_container export_payments() const;
     void import_payments(const payment_container &payments);
@@ -1883,11 +1918,13 @@ private:
     ExportFormat m_export_format;
     bool m_load_deprecated_formats;
 
+    bool m_has_ever_refreshed_from_node;
+
     static boost::mutex default_daemon_address_lock;
     static std::string default_daemon_address;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 29)
+BOOST_CLASS_VERSION(tools::wallet2, 30)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 12)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)
@@ -1898,7 +1935,7 @@ BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8)
 BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6)
 BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 18)
 BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0)
-BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0)
+BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4)
 BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3)
@@ -1908,6 +1945,17 @@ namespace boost
 {
   namespace serialization
   {
+    template<class Archive, class F, class S, class T>
+    inline void serialize(
+        Archive & ar,
+        std::tuple<F, S, T> & t,
+        const unsigned int /* file_version */
+    ){
+        ar & boost::serialization::make_nvp("f", std::get<0>(t));
+        ar & boost::serialization::make_nvp("s", std::get<1>(t));
+        ar & boost::serialization::make_nvp("t", std::get<2>(t));
+    }
+
     template <class Archive>
     inline typename std::enable_if<!Archive::is_loading::value, void>::type initialize_transfer_details(Archive &a, tools::wallet2::transfer_details &x, const boost::serialization::version_type ver)
     {
@@ -2268,7 +2316,17 @@ namespace boost
     inline void serialize(Archive &a, tools::wallet2::unsigned_tx_set &x, const boost::serialization::version_type ver)
     {
       a & x.txes;
-      a & x.transfers;
+      if (ver == 0)
+      {
+        // load old version
+        std::pair<size_t, tools::wallet2::transfer_container> old_transfers;
+        a & old_transfers;
+        std::get<0>(x.transfers) = std::get<0>(old_transfers);
+        std::get<1>(x.transfers) = std::get<0>(old_transfers) + std::get<1>(old_transfers).size();
+        std::get<2>(x.transfers) = std::get<1>(old_transfers);
+        return;
+      }
+      throw std::runtime_error("Boost serialization not supported for newest unsigned_tx_set");
     }
 
     template <class Archive>

src/wallet/wallet_rpc_server.cpp

@@ -2792,7 +2792,7 @@ namespace tools
 
     try
     {
-      res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str(req.all));
+      res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str(req.all, req.start, req.count));
     }
     catch (const std::exception &e)
     {

src/wallet/wallet_rpc_server_commands_defs.h

@@ -1785,9 +1785,13 @@ namespace wallet_rpc
     struct request_t
     {
       bool all;
+      uint32_t start;
+      uint32_t count;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(all)
+        KV_SERIALIZE_OPT(start, 0u)
+        KV_SERIALIZE_OPT(count, 0xffffffffu)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;

tests/functional_tests/cold_signing.py

@@ -34,13 +34,22 @@
 from __future__ import print_function
 from framework.daemon import Daemon
 from framework.wallet import Wallet
+import random
+
+SEED = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
+STANDARD_ADDRESS = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+SUBADDRESS = '84QRUYawRNrU3NN1VpFRndSukeyEb3Xpv8qZjjsoJZnTYpDYceuUTpog13D7qPxpviS7J29bSgSkR11hFFoXWk2yNdsR9WF'
 
 class ColdSigningTest():
     def run_test(self):
         self.reset()
         self.create(0)
         self.mine()
-        self.transfer()
+        for piecemeal_output_export in [False, True]:
+            self.transfer(piecemeal_output_export)
+        for piecemeal_output_export in [False, True]:
+            self.self_transfer_to_subaddress(piecemeal_output_export)
+        self.transfer_after_empty_export_import()
 
     def reset(self):
         print('Resetting blockchain')
@@ -57,17 +66,15 @@ class ColdSigningTest():
         try: self.hot_wallet.close_wallet()
         except: pass
 
-        self.cold_wallet = Wallet(idx = 1)
+        self.cold_wallet = Wallet(idx = 5)
         # close the wallet if any, will throw if none is loaded
         try: self.cold_wallet.close_wallet()
         except: pass
 
-        seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
+        res = self.cold_wallet.restore_deterministic_wallet(seed = SEED)
-        res = self.cold_wallet.restore_deterministic_wallet(seed = seed)
-        self.cold_wallet.set_daemon('127.0.0.1:11111', ssl_support = "disabled")
         spend_key = self.cold_wallet.query_key("spend_key").key
         view_key = self.cold_wallet.query_key("view_key").key
-        res = self.hot_wallet.generate_from_keys(viewkey = view_key, address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm')
+        res = self.hot_wallet.generate_from_keys(viewkey = view_key, address = STANDARD_ADDRESS)
 
         ok = False
         try: res = self.hot_wallet.query_key("spend_key")
@@ -79,28 +86,62 @@ class ColdSigningTest():
         assert ok
         assert self.cold_wallet.query_key("view_key").key == view_key
         assert self.cold_wallet.get_address().address == self.hot_wallet.get_address().address
+        assert self.cold_wallet.get_address().address == STANDARD_ADDRESS
 
     def mine(self):
         print("Mining some blocks")
         daemon = Daemon()
         wallet = Wallet()
 
-        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
+        daemon.generateblocks(STANDARD_ADDRESS, 80)
         wallet.refresh()
 
-    def transfer(self):
+    def export_import(self, piecemeal_output_export):
-        daemon = Daemon()
-
-        print("Creating transaction in hot wallet")
-
-        dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000}
-
         self.hot_wallet.refresh()
+
+        if piecemeal_output_export:
+            res = self.hot_wallet.incoming_transfers()
+            num_outputs = len(res.transfers)
+            done = [False] * num_outputs
+            while len([x for x in done if not done[x]]) > 0:
+                start = int(random.random() * num_outputs)
+                if start == num_outputs:
+                    num_outputs -= 1
+                count = 1 + int(random.random() * 5)
+                res = self.hot_wallet.export_outputs(all = True, start = start, count = count)
+
+                # the hot wallet cannot import outputs
+                ok = False
+                try:
+                    self.hot_wallet.import_outputs(res.outputs_data_hex)
+                except:
+                    ok = True
+                assert ok
+
+                try:
+                    self.cold_wallet.import_outputs(res.outputs_data_hex)
+                except Exception as e:
+                    # this just means we selected later outputs first, without filling
+                    # new outputs first
+                    if 'Imported outputs omit more outputs that we know of' not in str(e):
+                        raise
+                for i in range(start, start + count):
+                    if i < len(done):
+                        done[i] = True
+        else:
             res = self.hot_wallet.export_outputs()
             self.cold_wallet.import_outputs(res.outputs_data_hex)
+
         res = self.cold_wallet.export_key_images(True)
         self.hot_wallet.import_key_images(res.signed_key_images, offset = res.offset)
 
+    def create_tx(self, destination_addr, piecemeal_output_export):
+        daemon = Daemon()
+
+        dst = {'address': destination_addr, 'amount': 1000000000000}
+
+        self.export_import(piecemeal_output_export)
+
         res = self.hot_wallet.transfer([dst], ring_size = 16, get_tx_key = False)
         assert len(res.tx_hash) == 32*2
         txid = res.tx_hash
@@ -125,11 +166,11 @@ class ColdSigningTest():
         assert desc.unlock_time == 0
         assert desc.payment_id in ['', '0000000000000000']
         assert desc.change_amount == desc.amount_in - 1000000000000 - fee
-        assert desc.change_address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+        assert desc.change_address == STANDARD_ADDRESS
         assert desc.fee == fee
         assert len(desc.recipients) == 1
         rec = desc.recipients[0]
-        assert rec.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+        assert rec.address == destination_addr
         assert rec.amount == 1000000000000
 
         res = self.cold_wallet.sign_transfer(unsigned_txset)
@@ -148,7 +189,7 @@ class ColdSigningTest():
         assert len([x for x in (res['pending'] if 'pending' in res else []) if x.txid == txid]) == 1
         assert len([x for x in (res['out'] if 'out' in res else []) if x.txid == txid]) == 0
 
-        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
+        daemon.generateblocks(STANDARD_ADDRESS, 1)
         self.hot_wallet.refresh()
 
         res = self.hot_wallet.get_transfers()
@@ -160,6 +201,47 @@ class ColdSigningTest():
         res = self.cold_wallet.get_tx_key(txid)
         assert len(res.tx_key) == 64
 
+        self.export_import(piecemeal_output_export)
+
+    def transfer(self, piecemeal_output_export):
+        print("Creating transaction in hot wallet")
+        self.create_tx(STANDARD_ADDRESS, piecemeal_output_export)
+
+        res = self.cold_wallet.get_address()
+        assert len(res['addresses']) == 1
+        assert res['addresses'][0].address == STANDARD_ADDRESS
+        assert res['addresses'][0].used
+
+        res = self.hot_wallet.get_address()
+        assert len(res['addresses']) == 1
+        assert res['addresses'][0].address == STANDARD_ADDRESS
+        assert res['addresses'][0].used
+
+    def self_transfer_to_subaddress(self, piecemeal_output_export):
+        print("Self-spending to subaddress in hot wallet")
+        self.create_tx(SUBADDRESS, piecemeal_output_export)
+
+        res = self.cold_wallet.get_address()
+        assert len(res['addresses']) == 2
+        assert res['addresses'][0].address == STANDARD_ADDRESS
+        assert res['addresses'][0].used
+        assert res['addresses'][1].address == SUBADDRESS
+        assert res['addresses'][1].used
+
+        res = self.hot_wallet.get_address()
+        assert len(res['addresses']) == 2
+        assert res['addresses'][0].address == STANDARD_ADDRESS
+        assert res['addresses'][0].used
+        assert res['addresses'][1].address == SUBADDRESS
+        assert res['addresses'][1].used
+
+    def transfer_after_empty_export_import(self):
+        print("Creating transaction in hot wallet after empty export & import")
+        start_len = len(self.hot_wallet.get_transfers()['in'])
+        self.export_import(False)
+        assert start_len == len(self.hot_wallet.get_transfers()['in'])
+        self.create_tx(STANDARD_ADDRESS, False)
+        assert start_len == len(self.hot_wallet.get_transfers()['in']) - 1
 
 class Guard:
     def __enter__(self):

tests/functional_tests/functional_tests_rpc.py

@@ -40,8 +40,9 @@ except:
 N_MONERODS = 4
 
 # 4 wallets connected to the main offline monerod
-# a wallet connected to the first local online monerod
+# 1 wallet connected to the first local online monerod
-N_WALLETS = 5
+# 1 offline wallet
+N_WALLETS = 6
 
 WALLET_DIRECTORY = builddir + "/functional-tests-directory"
 FUNCTIONAL_TESTS_DIRECTORY = builddir + "/tests/functional_tests"
@@ -61,6 +62,7 @@ wallet_extra = [
   ["--daemon-port", "18180"],
   ["--daemon-port", "18180"],
   ["--daemon-port", "18182"],
+  ["--offline"],
 ]
 
 command_lines = []

tests/fuzz/cold-outputs.cpp

@@ -50,7 +50,7 @@ BEGIN_INIT_SIMPLE_FUZZER()
 END_INIT_SIMPLE_FUZZER()
 
 BEGIN_SIMPLE_FUZZER()
-  std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
+  std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
   binary_archive<false> ar{{buf, len}};
   ::serialization::serialize(ar, outputs);
   size_t n_outputs = wallet->import_outputs(outputs);

utils/python-rpc/framework/wallet.py

@@ -763,10 +763,13 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(get_languages)
 
-    def export_outputs(self):
+    def export_outputs(self, all = False, start = 0, count = 0xffffffff):
         export_outputs = {
             'method': 'export_outputs',
             'params': {
+                'all': all,
+                'start': start,
+                'count': count,
             },
             'jsonrpc': '2.0', 
             'id': '0'