refactor syncing wallet with progress and switching connections

core/src/main/java/haveno/core/api/CoreWalletsService.java

@@ -178,7 +178,7 @@ class CoreWalletsService {
         verifyWalletsAreAvailable();
         verifyEncryptedWalletIsUnlocked();
         try {
-            return xmrWalletService.getWallet().relayTx(metadata);
+            return xmrWalletService.relayTx(metadata);
         } catch (Exception ex) {
             log.error("", ex);
             throw new IllegalStateException(ex);

core/src/main/java/haveno/core/api/XmrConnectionService.java

@@ -273,7 +273,11 @@ public final class XmrConnectionService {
     }
 
     public synchronized boolean requestSwitchToNextBestConnection() {
-        log.warn("Requesting switch to next best monerod, current monerod={}", getConnection() == null ? null : getConnection().getUri());
+        return requestSwitchToNextBestConnection(null);
+    }
+
+    public synchronized boolean requestSwitchToNextBestConnection(MoneroRpcConnection sourceConnection) {
+        log.warn("Requesting switch to next best monerod, source monerod={}", sourceConnection == null ? getConnection() == null ? null : getConnection().getUri() : sourceConnection.getUri());
 
         // skip if shut down started
         if (isShutDownStarted) {
@@ -281,9 +285,15 @@ public final class XmrConnectionService {
             return false;
         }
 
+        // skip if connection is already switched
+        if (sourceConnection != null && sourceConnection != getConnection()) {
+            log.warn("Skipping switch to next best Monero connection because source connection is not current connection");
+            return false;
+        }
+
         // skip if connection is fixed
         if (isFixedConnection() || !connectionManager.getAutoSwitch()) {
-            log.info("Skipping switch to next best Monero connection because connection is fixed or auto switch is disabled");
+            log.warn("Skipping switch to next best Monero connection because connection is fixed or auto switch is disabled");
             return false;
         }
 

core/src/main/java/haveno/core/app/WalletAppSetup.java

@@ -133,6 +133,22 @@ public class WalletAppSetup {
                     String result;
                     if (exception == null && errorMsg == null) {
                         
+                        // update daemon sync progress
+                        double chainDownloadPercentageD = xmrConnectionService.downloadPercentageProperty().doubleValue();
+                        Long bestChainHeight = xmrConnectionService.chainHeightProperty().get();
+                        String chainHeightAsString = bestChainHeight != null && bestChainHeight > 0 ? String.valueOf(bestChainHeight) : "";
+                        if (chainDownloadPercentageD < 1) {
+                            xmrDaemonSyncProgress.set(chainDownloadPercentageD);
+                            if (chainDownloadPercentageD > 0.0) {
+                                String synchronizingWith = Res.get("mainView.footer.xmrInfo.synchronizingWith", getXmrDaemonNetworkAsString(), chainHeightAsString, FormattingUtils.formatToRoundedPercentWithSymbol(chainDownloadPercentageD));
+                                result = Res.get("mainView.footer.xmrInfo", synchronizingWith, "");
+                            } else {
+                                result = Res.get("mainView.footer.xmrInfo",
+                                        Res.get("mainView.footer.xmrInfo.connectingTo"),
+                                        getXmrDaemonNetworkAsString());
+                            }
+                        } else {
+
                             // update wallet sync progress
                             double walletDownloadPercentageD = (double) walletDownloadPercentage;
                             xmrWalletSyncProgress.set(walletDownloadPercentageD);
@@ -144,29 +160,15 @@ public class WalletAppSetup {
                                 result = Res.get("mainView.footer.xmrInfo", synchronizedWith, feeInfo);
                                 getXmrSplashSyncIconId().set("image-connection-synced");
                                 downloadCompleteHandler.run();
-                        } else if (walletDownloadPercentageD > 0) {
+                            } else if (walletDownloadPercentageD >= 0) {
                                 String synchronizingWith = Res.get("mainView.footer.xmrInfo.synchronizingWalletWith", getXmrWalletNetworkAsString(), walletHeightAsString, FormattingUtils.formatToRoundedPercentWithSymbol(walletDownloadPercentageD));
                                 result = Res.get("mainView.footer.xmrInfo", synchronizingWith, "");
                                 getXmrSplashSyncIconId().set(""); // clear synced icon
                             } else {
-
-                            // update daemon sync progress
-                            double chainDownloadPercentageD = xmrConnectionService.downloadPercentageProperty().doubleValue();
-                            xmrDaemonSyncProgress.set(chainDownloadPercentageD);
-                            Long bestChainHeight = xmrConnectionService.chainHeightProperty().get();
-                            String chainHeightAsString = bestChainHeight != null && bestChainHeight > 0 ? String.valueOf(bestChainHeight) : "";
-                            if (chainDownloadPercentageD == 1) {
                                 String synchronizedWith = Res.get("mainView.footer.xmrInfo.connectedTo", getXmrDaemonNetworkAsString(), chainHeightAsString);
                                 String feeInfo = ""; // TODO: feeService.isFeeAvailable() returns true, disable
                                 result = Res.get("mainView.footer.xmrInfo", synchronizedWith, feeInfo);
                                 getXmrSplashSyncIconId().set("image-connection-synced");
-                            } else if (chainDownloadPercentageD > 0.0) {
-                                String synchronizingWith = Res.get("mainView.footer.xmrInfo.synchronizingWith", getXmrDaemonNetworkAsString(), chainHeightAsString, FormattingUtils.formatToRoundedPercentWithSymbol(chainDownloadPercentageD));
-                                result = Res.get("mainView.footer.xmrInfo", synchronizingWith, "");
-                            } else {
-                                result = Res.get("mainView.footer.xmrInfo",
-                                        Res.get("mainView.footer.xmrInfo.connectingTo"),
-                                        getXmrDaemonNetworkAsString());
                             }
                         }
                     } else {

core/src/main/java/haveno/core/offer/OpenOfferManager.java

@@ -110,6 +110,7 @@ import javafx.collections.FXCollections;
 import javafx.collections.ObservableList;
 import javax.annotation.Nullable;
 import lombok.Getter;
+import monero.common.MoneroRpcConnection;
 import monero.daemon.model.MoneroKeyImageSpentStatus;
 import monero.daemon.model.MoneroTx;
 import monero.wallet.model.MoneroIncomingTransfer;
@@ -1089,6 +1090,7 @@ public class OpenOfferManager implements PeerManager.Listener, DecryptedDirectMe
             synchronized (HavenoUtils.getWalletFunctionLock()) {
                 long startTime = System.currentTimeMillis();
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         log.info("Creating split output tx to fund offer {} at subaddress {}", openOffer.getShortId(), entry.getSubaddressIndex());
                         splitOutputTx = xmrWalletService.createTx(new MoneroTxConfig()
@@ -1101,8 +1103,8 @@ public class OpenOfferManager implements PeerManager.Listener, DecryptedDirectMe
                     } catch (Exception e) {
                         if (e.getMessage().contains("not enough")) throw e; // do not retry if not enough funds
                         log.warn("Error creating split output tx to fund offer, offerId={}, subaddress={}, attempt={}/{}, error={}", openOffer.getShortId(), entry.getSubaddressIndex(), i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage());
+                        xmrWalletService.handleWalletError(e, sourceConnection);
                         if (stopped || i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrConnectionService.isConnected()) xmrWalletService.requestSwitchToNextBestConnection();
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     }
                 }

core/src/main/java/haveno/core/offer/placeoffer/tasks/MakerReserveOfferFunds.java

@@ -32,6 +32,7 @@ import haveno.core.trade.protocol.TradeProtocol;
 import haveno.core.xmr.model.XmrAddressEntry;
 import haveno.core.xmr.wallet.XmrWalletService;
 import lombok.extern.slf4j.Slf4j;
+import monero.common.MoneroRpcConnection;
 import monero.daemon.model.MoneroOutput;
 import monero.wallet.model.MoneroTxWallet;
 
@@ -82,14 +83,16 @@ public class MakerReserveOfferFunds extends Task<PlaceOfferModel> {
                 try {
                     synchronized (HavenoUtils.getWalletFunctionLock()) {
                         for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                            MoneroRpcConnection sourceConnection = model.getXmrWalletService().getConnectionService().getConnection();
                             try {
                                 //if (true) throw new RuntimeException("Pretend error");
                                 reserveTx = model.getXmrWalletService().createReserveTx(penaltyFee, makerFee, sendAmount, securityDeposit, returnAddress, openOffer.isReserveExactAmount(), preferredSubaddressIndex);
                             } catch (Exception e) {
                                 log.warn("Error creating reserve tx, offerId={}, attempt={}/{}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, openOffer.getShortId(), e.getMessage());
+                                model.getXmrWalletService().handleWalletError(e, sourceConnection);
+                                verifyPending();
                                 if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
                                 model.getProtocol().startTimeoutTimer(); // reset protocol timeout
-                                if (model.getXmrWalletService().getConnectionService().isConnected()) model.getXmrWalletService().requestSwitchToNextBestConnection();
                                 HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                             }
         
@@ -129,7 +132,11 @@ public class MakerReserveOfferFunds extends Task<PlaceOfferModel> {
         }
     }
 
-    public void verifyPending() {
+    private boolean isPending() {
-        if (!model.getOpenOffer().isPending()) throw new RuntimeException("Offer " + model.getOpenOffer().getOffer().getId() + " is canceled");
+        return model.getOpenOffer().isPending();
+    }
+
+    private void verifyPending() {
+        if (!isPending()) throw new RuntimeException("Offer " + model.getOpenOffer().getOffer().getId() + " is canceled");
     }
 }

core/src/main/java/haveno/core/support/dispute/arbitration/ArbitrationManager.java

@@ -78,6 +78,7 @@ import haveno.network.p2p.P2PService;
 import haveno.network.p2p.network.Connection;
 import haveno.network.p2p.network.MessageListener;
 import lombok.extern.slf4j.Slf4j;
+import monero.common.MoneroRpcConnection;
 import monero.wallet.MoneroWallet;
 import monero.wallet.model.MoneroDestination;
 import monero.wallet.model.MoneroMultisigSignResult;
@@ -501,6 +502,7 @@ public final class ArbitrationManager extends DisputeManager<ArbitrationDisputeL
 
         // submit fully signed payout tx to the network
         for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+            MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
             try {
                 List<String> txHashes = multisigWallet.submitMultisigTxHex(disputeTxSet.getMultisigTxHex());
                 disputeTxSet.getTxs().get(0).setHash(txHashes.get(0)); // manually update hash which is known after signed
@@ -509,7 +511,7 @@ public final class ArbitrationManager extends DisputeManager<ArbitrationDisputeL
                 if (trade.isPayoutPublished()) throw new IllegalStateException("Payout tx already published for " + trade.getClass().getSimpleName() + " " + trade.getShortId());
                 log.warn("Failed to submit dispute payout tx, tradeId={}, attempt={}/{}, error={}", trade.getShortId(), i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage());
                 if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                if (trade.getXmrConnectionService().isConnected()) trade.requestSwitchToNextBestConnection();
+                if (trade.getXmrConnectionService().isConnected()) trade.requestSwitchToNextBestConnection(sourceConnection);
                 HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
             }
         }

core/src/main/java/haveno/core/trade/HavenoUtils.java

@@ -506,4 +506,16 @@ public class HavenoUtils {
     public static void setTopError(String msg) {
         havenoSetup.getTopErrorMsg().set(msg);
     }
+
+    public static boolean isConnectionRefused(Exception e) {
+        return e != null && e.getMessage().contains("Connection refused");
+    }
+
+    public static boolean isReadTimeout(Exception e) {
+        return e != null && e.getMessage().contains("Read timed out");
+    }
+
+    public static boolean isUnresponsive(Exception e) {
+        return isConnectionRefused(e) || isReadTimeout(e);
+    }
 }

core/src/main/java/haveno/core/trade/Trade.java

@@ -850,8 +850,8 @@ public abstract class Trade implements Tradable, Model {
         }
     }
 
-    public boolean requestSwitchToNextBestConnection() {
+    public boolean requestSwitchToNextBestConnection(MoneroRpcConnection sourceConnection) {
-        if (xmrConnectionService.requestSwitchToNextBestConnection()) {
+        if (xmrConnectionService.requestSwitchToNextBestConnection(sourceConnection)) {
             onConnectionChanged(xmrConnectionService.getConnection()); // change connection on same thread
             return true;
         }
@@ -895,10 +895,6 @@ public abstract class Trade implements Tradable, Model {
         }).start();
     }
 
-    private boolean isReadTimeoutError(String errMsg) {
-        return errMsg.contains("Read timed out");
-    }
-
     // TODO: checking error strings isn't robust, but the library doesn't provide a way to check if multisig hex is invalid. throw IllegalArgumentException from library on invalid multisig hex?
     private boolean isInvalidImportError(String errMsg) {
         return errMsg.contains("Failed to parse hex") || errMsg.contains("Multisig info is for a different account");
@@ -1081,6 +1077,7 @@ public abstract class Trade implements Tradable, Model {
         synchronized (walletLock) {
             synchronized (HavenoUtils.getDaemonLock()) { // lock on daemon because import calls full refresh
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         doImportMultisigHex();
                         break;
@@ -1088,9 +1085,8 @@ public abstract class Trade implements Tradable, Model {
                         throw e;
                     } catch (Exception e) {
                         log.warn("Failed to import multisig hex, tradeId={}, attempt={}/{}, error={}", getShortId(), i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage());
+                        handleWalletError(e, sourceConnection);
                         if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection();
-                        if (isReadTimeoutError(e.getMessage())) forceRestartTradeWallet(); // wallet can be stuck a while
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     }
                 }
@@ -1168,6 +1164,12 @@ public abstract class Trade implements Tradable, Model {
         log.info("Done importing multisig hexes for {} {} in {} ms, count={}", getClass().getSimpleName(), getShortId(), System.currentTimeMillis() - startTime, multisigHexes.size());
     }
 
+    private void handleWalletError(Exception e, MoneroRpcConnection sourceConnection) {
+        if (HavenoUtils.isUnresponsive(e)) forceCloseWallet(); // wallet can be stuck a while
+        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(sourceConnection);
+        getWallet(); // re-open wallet
+    }
+
     private String getMultisigHexRole(String multisigHex) {
         if (multisigHex.equals(getArbitrator().getUpdatedMultisigHex())) return "arbitrator";
         if (multisigHex.equals(getBuyer().getUpdatedMultisigHex())) return "buyer";
@@ -1189,14 +1191,15 @@ public abstract class Trade implements Tradable, Model {
         synchronized (walletLock) {
             synchronized (HavenoUtils.getWalletFunctionLock()) {
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         return doCreatePayoutTx();
                     } catch (IllegalArgumentException | IllegalStateException e) {
                         throw e;
                     } catch (Exception e) {
                         log.warn("Failed to create payout tx, tradeId={}, attempt={}/{}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage());
+                        handleWalletError(e, sourceConnection);
                         if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection();
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     }
                 }
@@ -1246,6 +1249,7 @@ public abstract class Trade implements Tradable, Model {
         synchronized (walletLock) {
             synchronized (HavenoUtils.getWalletFunctionLock()) {
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         if (wallet.isMultisigImportNeeded()) throw new IllegalStateException("Cannot create dispute payout tx because multisig import is needed for " + getClass().getSimpleName() + " " + getShortId());
                         return createTx(txConfig);
@@ -1254,8 +1258,8 @@ public abstract class Trade implements Tradable, Model {
                     } catch (Exception e) {
                         if (e.getMessage().contains("not possible")) throw new IllegalArgumentException("Loser payout is too small to cover the mining fee");
                         log.warn("Failed to create dispute payout tx, tradeId={}, attempt={}/{}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage());
+                        handleWalletError(e, sourceConnection);
                         if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection();
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     }
                 }
@@ -1275,6 +1279,7 @@ public abstract class Trade implements Tradable, Model {
         synchronized (walletLock) {
             synchronized (HavenoUtils.getWalletFunctionLock()) {
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         doProcessPayoutTx(payoutTxHex, sign, publish);
                         break;
@@ -1282,8 +1287,8 @@ public abstract class Trade implements Tradable, Model {
                         throw e;
                     } catch (Exception e) {
                         log.warn("Failed to process payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage());
+                        handleWalletError(e, sourceConnection);
                         if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection();
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     } finally {
                         requestSaveWallet();
@@ -2412,6 +2417,7 @@ public abstract class Trade implements Tradable, Model {
     }
 
     private void syncWallet(boolean pollWallet) {
+        MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
         try {
             if (getWallet() == null) throw new RuntimeException("Cannot sync trade wallet because it doesn't exist for " + getClass().getSimpleName() + ", " + getId());
             if (getWallet().getDaemonConnection() == null) throw new RuntimeException("Cannot sync trade wallet because it's not connected to a Monero daemon for " + getClass().getSimpleName() + ", " + getId());
@@ -2432,7 +2438,7 @@ public abstract class Trade implements Tradable, Model {
     
             if (pollWallet) pollWallet();
         } catch (Exception e) {
-            ThreadUtils.execute(() -> requestSwitchToNextBestConnection(), getId());
+            ThreadUtils.execute(() -> requestSwitchToNextBestConnection(sourceConnection), getId());
             throw e;
         }
     }
@@ -2561,11 +2567,12 @@ public abstract class Trade implements Tradable, Model {
 
                 // rescan spent outputs to detect unconfirmed payout tx
                 if (isPayoutExpected && wallet.getBalance().compareTo(BigInteger.ZERO) > 0) {
+                    MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                     try {
                         wallet.rescanSpent();
                     } catch (Exception e) {
                         log.warn("Failed to rescan spent outputs for {} {}, errorMessage={}", getClass().getSimpleName(), getShortId(), e.getMessage());
-                        ThreadUtils.execute(() -> requestSwitchToNextBestConnection(), getId()); // do not block polling thread
+                        ThreadUtils.execute(() -> requestSwitchToNextBestConnection(sourceConnection), getId()); // do not block polling thread
                     }
                 }
 
@@ -2610,12 +2617,11 @@ public abstract class Trade implements Tradable, Model {
                 }
             }
         } catch (Exception e) {
-            boolean isConnectionRefused = e.getMessage() != null && e.getMessage().contains("Connection refused");
+            if (HavenoUtils.isUnresponsive(e)) forceRestartTradeWallet();
-            if (isConnectionRefused) forceRestartTradeWallet();
             else {
                 boolean isWalletConnected = isWalletConnectedToDaemon();
                 if (wallet != null && !isShutDownStarted && isWalletConnected) {
-                    log.warn("Error polling trade wallet for {} {}, errorMessage={}. Monerod={}", getClass().getSimpleName(), getShortId(), e.getMessage(), getXmrWalletService().getConnectionService().getConnection());
+                    log.warn("Error polling trade wallet for {} {}, errorMessage={}. Monerod={}", getClass().getSimpleName(), getShortId(), e.getMessage(), wallet.getDaemonConnection());
                     //e.printStackTrace();
                 }
             }
@@ -2675,7 +2681,8 @@ public abstract class Trade implements Tradable, Model {
                         log.warn("Rescanning blockchain for {} {}", getClass().getSimpleName(), getShortId());
                         wallet.rescanBlockchain();
                     } catch (Exception e) {
-                        if (isReadTimeoutError(e.getMessage())) forceRestartTradeWallet(); // wallet can be stuck a while
+                        log.warn("Error rescanning blockchain for {} {}, errorMessage={}", getClass().getSimpleName(), getShortId(), e.getMessage());
+                        if (HavenoUtils.isUnresponsive(e)) forceRestartTradeWallet(); // wallet can be stuck a while
                         throw e;
                     } finally {
 

core/src/main/java/haveno/core/trade/protocol/tasks/MaybeSendSignContractRequest.java

@@ -31,6 +31,7 @@ import haveno.core.xmr.model.XmrAddressEntry;
 import haveno.core.xmr.wallet.XmrWalletService;
 import haveno.network.p2p.SendDirectMessageListener;
 import lombok.extern.slf4j.Slf4j;
+import monero.common.MoneroRpcConnection;
 import monero.wallet.model.MoneroTxWallet;
 
 import java.math.BigInteger;
@@ -100,12 +101,14 @@ public class MaybeSendSignContractRequest extends TradeTask {
                 try {
                     synchronized (HavenoUtils.getWalletFunctionLock()) {
                         for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                            MoneroRpcConnection sourceConnection = trade.getXmrConnectionService().getConnection();
                             try {
                                 depositTx = trade.getXmrWalletService().createDepositTx(trade, reserveExactAmount, subaddressIndex);
                             } catch (Exception e) {
                                 log.warn("Error creating deposit tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, trade.getShortId(), e.getMessage());
+                                trade.getXmrWalletService().handleWalletError(e, sourceConnection);
+                                if (isTimedOut()) throw new RuntimeException("Trade protocol has timed out while creating deposit tx, tradeId=" + trade.getShortId());
                                 if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                                if (trade.getXmrConnectionService().isConnected()) trade.getXmrWalletService().requestSwitchToNextBestConnection();
                                 HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                             }
             

core/src/main/java/haveno/core/trade/protocol/tasks/TakerReserveTradeFunds.java

@@ -26,6 +26,7 @@ import haveno.core.trade.protocol.TradeProtocol;
 import haveno.core.xmr.model.XmrAddressEntry;
 import haveno.core.xmr.wallet.XmrWalletService;
 import lombok.extern.slf4j.Slf4j;
+import monero.common.MoneroRpcConnection;
 import monero.wallet.model.MoneroTxWallet;
 
 import java.math.BigInteger;
@@ -66,12 +67,14 @@ public class TakerReserveTradeFunds extends TradeTask {
                 try {
                     synchronized (HavenoUtils.getWalletFunctionLock()) {
                         for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                            MoneroRpcConnection sourceConnection = trade.getXmrConnectionService().getConnection();
                             try {
                                 reserveTx = model.getXmrWalletService().createReserveTx(penaltyFee, takerFee, sendAmount, securityDeposit, returnAddress, false, null);
                             } catch (Exception e) {
                                 log.warn("Error creating reserve tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, trade.getShortId(), e.getMessage());
+                                trade.getXmrWalletService().handleWalletError(e, sourceConnection);
+                                if (isTimedOut()) throw new RuntimeException("Trade protocol has timed out while creating reserve tx, tradeId=" + trade.getShortId());
                                 if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                                if (trade.getXmrConnectionService().isConnected()) trade.getXmrWalletService().requestSwitchToNextBestConnection();
                                 HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                             }
             

core/src/main/java/haveno/core/xmr/wallet/XmrWalletService.java

@@ -160,9 +160,11 @@ public class XmrWalletService {
     private boolean isClosingWallet;
     private boolean isShutDownStarted;
     private ExecutorService syncWalletThreadPool = Executors.newFixedThreadPool(10); // TODO: adjust based on connection type
+    private boolean isSyncingWithProgress;
     private Long syncStartHeight;
     private TaskLooper syncProgressLooper;
     private CountDownLatch syncProgressLatch;
+    private Exception syncProgressError;
     private Timer syncProgressTimeout;
     private static final int SYNC_PROGRESS_TIMEOUT_SECONDS = 60;
 
@@ -178,7 +180,9 @@ public class XmrWalletService {
     private List<MoneroSubaddress> cachedSubaddresses;
     private List<MoneroOutputWallet> cachedOutputs;
     private List<MoneroTxWallet> cachedTxs;
-    private boolean runReconnectTestOnStartup = false; // test reconnecting on startup while syncing so the wallet is blocked
+    private boolean testReconnectOnStartup = false; // test reconnecting on startup while syncing so the wallet is blocked
+    private String testReconnectMonerod1 = "http://node.community.rino.io:18081";
+    private String testReconnectMonerod2 = "http://nodex.monerujo.io:18081";
 
     @SuppressWarnings("unused")
     @Inject
@@ -473,6 +477,14 @@ public class XmrWalletService {
         }
     }
 
+    public String relayTx(String metadata) {
+        synchronized (WALLET_LOCK) {
+            String txId = wallet.relayTx(metadata);
+            requestSaveMainWallet();
+            return txId;
+        }
+    }
+
     public MoneroTxWallet createTx(List<MoneroDestination> destinations) {
         MoneroTxWallet tx = createTx(new MoneroTxConfig().setAccountIndex(0).setDestinations(destinations).setRelay(false).setCanSplit(false));
         //printTxs("XmrWalletService.createTx", tx);
@@ -1289,9 +1301,19 @@ public class XmrWalletService {
         else log.info(appliedMsg);
 
         // listen for connection changes
-        xmrConnectionService.addConnectionListener(connection -> ThreadUtils.execute(() -> {
+        xmrConnectionService.addConnectionListener(connection -> {
+            if (wasWalletSynced && !isSyncingWithProgress) {
+                ThreadUtils.execute(() -> {
                     onConnectionChanged(connection);
-        }, THREAD_ID));
+                 }, THREAD_ID);
+            } else {
+
+                // force restart main wallet if connection changed while syncing
+                log.warn("Force restarting main wallet because connection changed while syncing");
+                forceRestartMainWallet();
+                return;
+            }
+        });
 
         // initialize main wallet when daemon synced
         walletInitListener = (obs, oldVal, newVal) -> initMainWalletIfConnected();
@@ -1340,6 +1362,7 @@ public class XmrWalletService {
                     long date = localDateTime.toEpochSecond(ZoneOffset.UTC);
                     user.setWalletCreationDate(date);
                 }
+                walletHeight.set(wallet.getHeight());
                 isClosingWallet = false;
             }
 
@@ -1348,6 +1371,7 @@ public class XmrWalletService {
                 log.info("Monero wallet path={}", wallet.getPath());
 
                 // sync main wallet if applicable
+                // TODO: error handling and re-initialization is jenky, refactor
                 if (sync && numAttempts > 0) {
                     try {
 
@@ -1360,7 +1384,16 @@ public class XmrWalletService {
                         // sync main wallet
                         log.info("Syncing main wallet");
                         long time = System.currentTimeMillis();
+                        MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
+                        try {
                             syncWithProgress(); // blocking
+                        } catch (Exception e) {
+                            log.warn("Error syncing wallet with progress on startup: " + e.getMessage());
+                            forceCloseMainWallet();
+                            requestSwitchToNextBestConnection(sourceConnection);
+                            maybeInitMainWallet(true, numAttempts - 1); // re-initialize wallet and sync again
+                            return;
+                        }
                         log.info("Done syncing main wallet in " + (System.currentTimeMillis() - time) + " ms");
 
                         // poll wallet
@@ -1435,69 +1468,83 @@ public class XmrWalletService {
     }
 
     private void syncWithProgress() {
+        synchronized (WALLET_LOCK) {
 
-        // start sync progress timeout
+            // set initial state
-        resetSyncProgressTimeout();
+            isSyncingWithProgress = true;
-
+            syncProgressError = null;
-        // show sync progress
+            updateSyncProgress(walletHeight.get());
-        updateSyncProgress(wallet.getHeight());
 
             // test connection changing on startup before wallet synced
-        if (runReconnectTestOnStartup) {
+            if (testReconnectOnStartup) {
                 UserThread.runAfter(() -> {
                     log.warn("Testing connection change on startup before wallet synced");
-                xmrConnectionService.setConnection("http://node.community.rino.io:18081"); // TODO: needs to be online
+                    if (xmrConnectionService.getConnection().getUri().equals(testReconnectMonerod1)) xmrConnectionService.setConnection(testReconnectMonerod2);
+                    else xmrConnectionService.setConnection(testReconnectMonerod1);
                 }, 1);
-            runReconnectTestOnStartup = false; // only run once
+                testReconnectOnStartup = false; // only run once
             }
 
-        // get sync notifications from native wallet
+            // native wallet provides sync notifications
             if (wallet instanceof MoneroWalletFull) {
-            if (runReconnectTestOnStartup) HavenoUtils.waitFor(1000); // delay sync to test
+                if (testReconnectOnStartup) HavenoUtils.waitFor(1000); // delay sync to test
                 wallet.sync(new MoneroWalletListener() {
                     @Override
                     public void onSyncProgress(long height, long startHeight, long endHeight, double percentDone, String message) {
                         updateSyncProgress(height);
                     }
                 });
-            wasWalletSynced = true;
+                setWalletSyncedWithProgress();
                 return;
             }
 
-        // poll wallet for progress
+            // start polling wallet for progress
-        wallet.startSyncing(xmrConnectionService.getRefreshPeriodMs());
             syncProgressLatch = new CountDownLatch(1);
             syncProgressLooper = new TaskLooper(() -> {
                 if (wallet == null) return;
-            long height = 0;
+                long height;
                 try {
                     height = wallet.getHeight(); // can get read timeout while syncing
                 } catch (Exception e) {
-                if (!isShutDownStarted) e.printStackTrace();
+                    log.warn("Error getting wallet height while syncing with progress: " + e.getMessage());
+                    if (wallet != null && !isShutDownStarted) e.printStackTrace();
+
+                    // stop polling and release latch
+                    syncProgressError = e;
+                    syncProgressLatch.countDown();
                     return;
                 }
-            if (height < xmrConnectionService.getTargetHeight()) updateSyncProgress(height);
-            else {
-                syncProgressLooper.stop();
-                wasWalletSynced = true;
                 updateSyncProgress(height);
+                if (height >= xmrConnectionService.getTargetHeight()) {
+                    setWalletSyncedWithProgress();
                     syncProgressLatch.countDown();
                 }
             });
+            wallet.startSyncing(xmrConnectionService.getRefreshPeriodMs());
             syncProgressLooper.start(1000);
+
+            // wait for sync to complete
             HavenoUtils.awaitLatch(syncProgressLatch);
-        wallet.stopSyncing();
+
-        if (!wasWalletSynced) throw new IllegalStateException("Failed to sync wallet with progress");
+            // stop polling
+            syncProgressLooper.stop();
+            syncProgressTimeout.stop();
+            if (wallet != null) wallet.stopSyncing(); // can become null if interrupted by force close
+            isSyncingWithProgress = false;
+            if (syncProgressError != null) throw new RuntimeException(syncProgressError);
+        }
     }
 
     private void updateSyncProgress(long height) {
-        UserThread.execute(() -> {
-            walletHeight.set(height);
         resetSyncProgressTimeout();
+        UserThread.execute(() -> {
+
+            // set wallet height
+            walletHeight.set(height);
 
             // new wallet reports height 1 before synced
             if (height == 1) {
-                downloadListener.progress(.0001, xmrConnectionService.getTargetHeight() - height, null); // >0% shows progress bar
+                downloadListener.progress(0, xmrConnectionService.getTargetHeight() - height, null);
                 return;
             }
 
@@ -1505,7 +1552,7 @@ public class XmrWalletService {
             long targetHeight = xmrConnectionService.getTargetHeight();
             long blocksLeft = targetHeight - walletHeight.get();
             if (syncStartHeight == null) syncStartHeight = walletHeight.get();
-            double percent = Math.min(1.0, targetHeight == syncStartHeight ? 1.0 : ((double) Math.max(1, (double) walletHeight.get() - syncStartHeight) / (double) (targetHeight - syncStartHeight))); // grant at least 1 block to show progress
+            double percent = Math.min(1.0, targetHeight == syncStartHeight ? 1.0 : ((double) walletHeight.get() - syncStartHeight) / (double) (targetHeight - syncStartHeight));
             downloadListener.progress(percent, blocksLeft, null);
         });
     }
@@ -1513,15 +1560,18 @@ public class XmrWalletService {
     private synchronized void resetSyncProgressTimeout() {
         if (syncProgressTimeout != null) syncProgressTimeout.stop();
         syncProgressTimeout = UserThread.runAfter(() -> {
-            if (isShutDownStarted || wasWalletSynced) return;
+            if (isShutDownStarted) return;
-            log.warn("Sync progress timeout called");
+            syncProgressError = new RuntimeException("Sync progress timeout called");
-            forceCloseMainWallet();
+            syncProgressLatch.countDown();
-            requestSwitchToNextBestConnection();
-            maybeInitMainWallet(true);
-            resetSyncProgressTimeout();
         }, SYNC_PROGRESS_TIMEOUT_SECONDS, TimeUnit.SECONDS);
     }
 
+    private void setWalletSyncedWithProgress() {
+        wasWalletSynced = true;
+        isSyncingWithProgress = false;
+        syncProgressTimeout.stop();
+    }
+
     private MoneroWalletFull createWalletFull(MoneroWalletConfig config) {
 
         // must be connected to daemon
@@ -1686,14 +1736,6 @@ public class XmrWalletService {
             String newProxyUri = connection == null ? null : connection.getProxyUri();
             log.info("Setting daemon connection for main wallet, monerod={}, proxyUri={}", connection == null ? null : connection.getUri(), newProxyUri);
 
-            // force restart main wallet if connection changed before synced
-            if (!wasWalletSynced) {
-                if (!Boolean.TRUE.equals(xmrConnectionService.isConnected())) return;
-                log.warn("Force restarting main wallet because connection changed before inital sync");
-                forceRestartMainWallet();
-                return;
-            }
-
             // update connection
             if (wallet instanceof MoneroWalletRpc) {
                 if (StringUtils.equals(oldProxyUri, newProxyUri)) {
@@ -1702,7 +1744,7 @@ public class XmrWalletService {
                     log.info("Restarting main wallet because proxy URI has changed, old={}, new={}", oldProxyUri, newProxyUri); // TODO: set proxy without restarting wallet
                     closeMainWallet(true);
                     doMaybeInitMainWallet(false, MAX_SYNC_ATTEMPTS);
-                    return; // wallet is re-initialized
+                    return; // wallet re-initializes off thread
                 }
             } else {
                 wallet.setDaemonConnection(connection);
@@ -1771,19 +1813,26 @@ public class XmrWalletService {
 
     private void forceCloseMainWallet() {
         stopPolling();
-        if (wallet != null) {
+        if (wallet != null && !isClosingWallet) {
             isClosingWallet = true;
             forceCloseWallet(wallet, getWalletPath(MONERO_WALLET_NAME));
             wallet = null;
         }
     }
 
-    private void forceRestartMainWallet() {
+    public void forceRestartMainWallet() {
         log.warn("Force restarting main wallet");
+        if (isClosingWallet) return;
         forceCloseMainWallet();
         maybeInitMainWallet(true);
     }
 
+    public void handleWalletError(Exception e, MoneroRpcConnection sourceConnection) {
+        if (HavenoUtils.isUnresponsive(e)) forceCloseMainWallet(); // wallet can be stuck a while
+        if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(sourceConnection);
+        getWallet(); // re-open wallet
+    }
+
     private void startPolling() {
         synchronized (WALLET_LOCK) {
             if (isShutDownStarted || isPolling()) return;
@@ -1849,17 +1898,10 @@ public class XmrWalletService {
                 log.warn("Monero daemon is not synced within tolerance, height={}, targetHeight={}", xmrConnectionService.chainHeightProperty().get(), xmrConnectionService.getTargetHeight());
                 return;
             }
-
-            // switch to best connection if wallet is too far behind
-            if (wasWalletSynced && walletHeight.get() < xmrConnectionService.getTargetHeight() - NUM_BLOCKS_BEHIND_TOLERANCE && !Config.baseCurrencyNetwork().isTestnet()) {
-                log.warn("Updating connection because main wallet is {} blocks behind monerod, wallet height={}, monerod height={}", xmrConnectionService.getTargetHeight() - walletHeight.get(), walletHeight.get(), lastInfo.getHeight());
-                if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection();
-            }
-
             // sync wallet if behind daemon
             if (walletHeight.get() < xmrConnectionService.getTargetHeight()) {
                 synchronized (WALLET_LOCK) { // avoid long sync from blocking other operations
-                    syncMainWallet();
+                    syncWithProgress();
                 }
             }
 
@@ -1868,13 +1910,17 @@ public class XmrWalletService {
             if (updateTxs) {
                 synchronized (WALLET_LOCK) { // avoid long fetch from blocking other operations
                     synchronized (HavenoUtils.getDaemonLock()) {
+                        MoneroRpcConnection sourceConnection = xmrConnectionService.getConnection();
                         try {
                             cachedTxs = wallet.getTxs(new MoneroTxQuery().setIncludeOutputs(true));
                         } catch (Exception e) { // fetch from pool can fail
                             if (!isShutDownStarted) {
-                                if (lastLogPollErrorTimestamp == null || System.currentTimeMillis() - lastLogPollErrorTimestamp > HavenoUtils.LOG_POLL_ERROR_PERIOD_MS) { // limit error logging
+
+                                // throttle error handling
+                                if (lastLogPollErrorTimestamp == null || System.currentTimeMillis() - lastLogPollErrorTimestamp > HavenoUtils.LOG_POLL_ERROR_PERIOD_MS) {
                                     log.warn("Error polling main wallet's transactions from the pool: {}", e.getMessage());
                                     lastLogPollErrorTimestamp = System.currentTimeMillis();
+                                    requestSwitchToNextBestConnection(sourceConnection);
                                 }
                             }
                         }
@@ -1883,8 +1929,7 @@ public class XmrWalletService {
             }
         } catch (Exception e) {
             if (wallet == null || isShutDownStarted) return;
-            boolean isConnectionRefused = e.getMessage() != null && e.getMessage().contains("Connection refused");
+            if (HavenoUtils.isUnresponsive(e)) forceRestartMainWallet();
-            if (isConnectionRefused) forceRestartMainWallet();
             else if (isWalletConnectedToDaemon()) {
                 log.warn("Error polling main wallet, errorMessage={}. Monerod={}", e.getMessage(), getConnectionService().getConnection());
                 //e.printStackTrace();
@@ -1927,8 +1972,12 @@ public class XmrWalletService {
         }
     }
 
-    public boolean requestSwitchToNextBestConnection() {
+    private boolean requestSwitchToNextBestConnection() {
-        return xmrConnectionService.requestSwitchToNextBestConnection();
+        return requestSwitchToNextBestConnection(null);
+    }
+
+    public boolean requestSwitchToNextBestConnection(MoneroRpcConnection sourceConnection) {
+        return xmrConnectionService.requestSwitchToNextBestConnection(sourceConnection);
     }
 
     private void onNewBlock(long height) {

desktop/src/main/java/haveno/desktop/main/funds/withdrawal/WithdrawalView.java

@@ -65,6 +65,7 @@ import javafx.scene.control.Button;
 import javafx.scene.layout.GridPane;
 import javafx.scene.layout.StackPane;
 import javafx.scene.layout.VBox;
+import monero.common.MoneroRpcConnection;
 import monero.common.MoneroUtils;
 import monero.wallet.model.MoneroTxConfig;
 import monero.wallet.model.MoneroTxWallet;
@@ -256,6 +257,7 @@ public class WithdrawalView extends ActivatableView<VBox, Void> {
                 // create tx
                 MoneroTxWallet tx = null;
                 for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) {
+                    MoneroRpcConnection sourceConnection = xmrWalletService.getConnectionService().getConnection();
                     try {
                         log.info("Creating withdraw tx");
                         long startTime = System.currentTimeMillis();
@@ -270,7 +272,7 @@ public class WithdrawalView extends ActivatableView<VBox, Void> {
                         if (isNotEnoughMoney(e.getMessage())) throw e;
                         log.warn("Error creating creating withdraw tx, attempt={}/{}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage());
                         if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e;
-                        if (xmrWalletService.getConnectionService().isConnected()) xmrWalletService.requestSwitchToNextBestConnection();
+                        if (xmrWalletService.getConnectionService().isConnected()) xmrWalletService.requestSwitchToNextBestConnection(sourceConnection);
                         HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying
                     }
                 }