preparescript: --usebtcfastsync option will initialise the BTC datadir from a chain snapshot.

.cirrus.yml

@@ -8,7 +8,7 @@ lint_task:
   script:
     - flake8 --version
     - PYTHONWARNINGS="ignore" flake8 --ignore=E501,F841,W503 --exclude=basicswap/contrib,messages_pb2.py,.eggs,.tox
-    - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,gitianpubkeys,*.pyc,*basicswap/contrib,*mnemonics.py
+    - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,pgp,*.pyc,*basicswap/contrib,*mnemonics.py
 
 test_task:
   environment:

.travis.yml

@@ -52,7 +52,7 @@ jobs:
       before_script:
       script:
         - PYTHONWARNINGS="ignore" flake8 --ignore=E501,F841,W503 --exclude=basicswap/contrib,messages_pb2.py,.eggs,.tox
-        - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,gitianpubkeys,*.pyc,*basicswap/contrib,*mnemonics.py
+        - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,pgp,*.pyc,*basicswap/contrib,*mnemonics.py
       after_success:
         - echo "End lint"
     - stage: test

basicswap/basicswap.py

@@ -608,12 +608,12 @@ class BasicSwap(BaseApp):
             if self.coin_clients[c]['connection_type'] == 'rpc' and chain_client_settings['manage_daemon'] is True:
                 self.stopDaemon(c)
 
-    def waitForDaemonRPC(self, coin_type):
+    def waitForDaemonRPC(self, coin_type, with_wallet=True):
         for i in range(self.startup_tries):
             if not self.is_running:
                 return
             try:
-                self.coin_clients[coin_type]['interface'].testDaemonRPC()
+                self.coin_clients[coin_type]['interface'].testDaemonRPC(with_wallet)
                 return
             except Exception as ex:
                 self.log.warning('Can\'t connect to %s RPC: %s.  Trying again in %d second/s.', coin_type, str(ex), (1 + i))
@@ -3778,12 +3778,11 @@ class BasicSwap(BaseApp):
                 self.mxDB.acquire()
                 use_session = scoped_session(self.session_factory)
 
-            link = session.query(AutomationLink).filter_by(active_ind=1, linked_type=TableTypes.OFFER, linked_id=offer.offer_id).first()
+            link = use_session.query(AutomationLink).filter_by(active_ind=1, linked_type=TableTypes.OFFER, linked_id=offer.offer_id).first()
-
             if not link:
                 return False
 
-            strategy = session.query(AutomationStrategy).filter_by(active_ind=1, record_id=link.strategy_id).first()
+            strategy = use_session.query(AutomationStrategy).filter_by(active_ind=1, record_id=link.strategy_id).first()
             opts = json.loads(strategy.data.decode('utf-8'))
 
             self.log.debug('Evaluating against strategy {}'.format(strategy.record_id))
@@ -3800,7 +3799,7 @@ class BasicSwap(BaseApp):
                 return False
 
             if strategy.only_known_identities:
-                identity_stats = session.query(KnownIdentity).filter_by(address=bid.bid_addr).first()
+                identity_stats = use_session.query(KnownIdentity).filter_by(address=bid.bid_addr).first()
                 if not identity_stats:
                     return False
 
@@ -3811,6 +3810,9 @@ class BasicSwap(BaseApp):
                     return False
 
             return True
+        except Exception as e:
+            self.log.error('shouldAutoAcceptBid: %s', str(e))
+            return False
         finally:
             if session is None:
                 use_session.close()

basicswap/interface_btc.py

@@ -211,8 +211,11 @@ class BTCInterface(CoinInterface):
         ensure(new_conf_target >= 1 and new_conf_target < 33, 'Invalid conf_target value')
         self._conf_target = new_conf_target
 
-    def testDaemonRPC(self):
+    def testDaemonRPC(self, with_wallet=True):
-        self.rpc_callback('getwalletinfo', [])
+        if with_wallet:
+            self.rpc_callback('getwalletinfo', [])
+        else:
+            self.rpc_callback('getblockchaininfo', [])
 
     def getDaemonVersion(self):
         return self.rpc_callback('getnetworkinfo')['version']

basicswap/interface_xmr.py

@@ -110,7 +110,7 @@ class XMRInterface(CoinInterface):
         with self._mx_wallet:
             self.rpc_wallet_cb('open_wallet', {'filename': self._wallet_filename})
 
-    def testDaemonRPC(self):
+    def testDaemonRPC(self, with_wallet=True):
         self.rpc_wallet_cb('get_languages')
 
     def getDaemonVersion(self):

bin/basicswap_prepare.py

@@ -117,7 +117,9 @@ TOR_DNS_PORT = int(os.getenv('TOR_DNS_PORT', 5353))
 TEST_TOR_PROXY = toBool(os.getenv('TEST_TOR_PROXY', 'true'))  # Expects a known exit node
 TEST_ONION_LINK = toBool(os.getenv('TEST_ONION_LINK', 'false'))
 
-extract_core_overwrite = True
+BITCOIN_FASTSYNC_URL = os.getenv('BITCOIN_FASTSYNC_URL', 'http://utxosets.blob.core.windows.net/public/')
+BITCOIN_FASTSYNC_FILE = os.getenv('BITCOIN_FASTSYNC_FILE', 'utxo-snapshot-bitcoin-mainnet-720179.tar')
+
 use_tor_proxy = False
 
 default_socket = socket.socket
@@ -199,9 +201,10 @@ def testOnionLink():
     logger.info('Onion links work.')
 
 
-def extractCore(coin, version_data, settings, bin_dir, release_path):
+def extractCore(coin, version_data, settings, bin_dir, release_path, extra_opts={}):
     version, version_tag, signers = version_data
     logger.info('extractCore %s v%s%s', coin, version, version_tag)
+    extract_core_overwrite = extra_opts.get('extract_core_overwrite', True)
 
     if coin == 'monero':
         bins = ['monerod', 'monero-wallet-rpc']
@@ -260,7 +263,7 @@ def extractCore(coin, version_data, settings, bin_dir, release_path):
                         logging.warning('Unable to set file permissions: %s, for %s', str(e), out_path)
 
 
-def prepareCore(coin, version_data, settings, data_dir):
+def prepareCore(coin, version_data, settings, data_dir, extra_opts={}):
     version, version_tag, signers = version_data
     logger.info('prepareCore %s v%s%s', coin, version, version_tag)
 
@@ -384,8 +387,8 @@ def prepareCore(coin, version_data, settings, data_dir):
 
             filename = '{}_{}.pgp'.format(coin, signing_key_name)
             pubkeyurls = (
-                'https://raw.githubusercontent.com/tecnovert/basicswap/master/gitianpubkeys/' + filename,
+                'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/keys/' + filename,
-                'https://gitlab.com/particl/basicswap/-/raw/master/gitianpubkeys/' + filename,
+                'https://gitlab.com/particl/basicswap/-/raw/master/pgp/keys/' + filename,
             )
             for url in pubkeyurls:
                 try:
@@ -405,7 +408,7 @@ def prepareCore(coin, version_data, settings, data_dir):
        and not (verified.status == 'signature valid' and verified.key_status == 'signing key has expired'):
         raise ValueError('Signature verification failed.')
 
-    extractCore(coin, version_data, settings, bin_dir, release_path)
+    extractCore(coin, version_data, settings, bin_dir, release_path, extra_opts)
 
 
 def writeTorSettings(fp, coin, coin_settings, tor_control_password):
@@ -424,10 +427,11 @@ def writeTorSettings(fp, coin, coin_settings, tor_control_password):
         fp.write(f'bind=0.0.0.0:{onionport}=onion\n')
 
 
-def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False, tor_control_password=None):
+def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
     core_settings = settings['chainclients'][coin]
     bin_dir = core_settings['bindir']
     data_dir = core_settings['datadir']
+    tor_control_password = extra_opts.get('tor_control_password', None)
 
     if not os.path.exists(data_dir):
         os.makedirs(data_dir)
@@ -466,7 +470,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False
         if os.path.exists(wallet_conf_path):
             exitWithError('{} exists'.format(wallet_conf_path))
         with open(wallet_conf_path, 'w') as fp:
-            if use_containers:
+            if extra_opts.get('use_containers', False) is True:
                 fp.write('daemon-address={}:{}\n'.format(core_settings['rpchost'], core_settings['rpcport']))
             fp.write('untrusted-daemon=1\n')
             fp.write('no-dns=1\n')
@@ -481,6 +485,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False
                 if not core_settings['manage_daemon']:
                     fp.write(f'proxy={TOR_PROXY_HOST}:{TOR_PROXY_PORT}\n')
         return
+
     core_conf_path = os.path.join(data_dir, coin + '.conf')
     if os.path.exists(core_conf_path):
         exitWithError('{} exists'.format(core_conf_path))
@@ -532,10 +537,43 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False
         else:
             logger.warning('Unknown coin %s', coin)
 
-    wallet_util = coin + '-wallet'
+    if coin == 'bitcoin' and extra_opts.get('use_btc_fastsync', False) is True:
-    if os.path.exists(os.path.join(bin_dir, wallet_util)):
+        logger.info('Initialising BTC chain with fastsync %s', BITCOIN_FASTSYNC_FILE)
-        logger.info('Creating wallet.dat for {}.'.format(wallet_util.capitalize()))
+        base_dir = extra_opts['data_dir']
-        callrpc_cli(bin_dir, data_dir, chain, '-wallet=wallet.dat create', wallet_util)
+        sync_file_path = os.path.join(base_dir, BITCOIN_FASTSYNC_FILE)
+        if not os.path.exists(sync_file_path):
+            sync_file_url = os.path.join(BITCOIN_FASTSYNC_URL, BITCOIN_FASTSYNC_FILE)
+            downloadFile(sync_file_url, sync_file_path)
+
+        asc_filename = BITCOIN_FASTSYNC_FILE + '.asc'
+        asc_file_path = os.path.join(base_dir, asc_filename)
+        if not os.path.exists(asc_file_path):
+            asc_file_urls = (
+                'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/sigs/' + asc_filename,
+                'https://gitlab.com/particl/basicswap/-/raw/master/pgp/sigs/' + asc_filename,
+            )
+            for url in asc_file_urls:
+                try:
+                    downloadFile(url, asc_file_path)
+                    break
+                except Exception as e:
+                    print('Download failed', e)
+        gpg = gnupg.GPG()
+        with open(asc_file_path, 'rb') as fp:
+            verified = gpg.verify_file(fp, sync_file_path)
+
+        if verified.valid is False \
+           and not (verified.status == 'signature valid' and verified.key_status == 'signing key has expired'):
+            raise ValueError('Signature verification failed.')
+
+        with tarfile.open(sync_file_path) as ft:
+            ft.extractall(path=data_dir)
+        # Create the wallet later, no option to set bestblock through wallet_util
+    else:
+        wallet_util = coin + '-wallet'
+        if os.path.exists(os.path.join(bin_dir, wallet_util)):
+            logger.info('Creating wallet.dat for {}.'.format(wallet_util.capitalize()))
+            callrpc_cli(bin_dir, data_dir, chain, '-wallet=wallet.dat create', wallet_util)
 
 
 def write_torrc(data_dir, tor_control_password):
@@ -696,13 +734,16 @@ def printHelp():
     logger.info('--htmlhost=              Interface to host on, default:127.0.0.1.')
     logger.info('--xmrrestoreheight=n     Block height to restore Monero wallet from, default:{}.'.format(DEFAULT_XMR_RESTORE_HEIGHT))
     logger.info('--noextractover          Prevent extracting cores if files exist.  Speeds up tests')
-    logger.info('--usetorproxy            Use TOR proxy.  Note that some download links may be inaccessible over TOR.')
+    logger.info('--usetorproxy            Use TOR proxy during setup.  Note that some download links may be inaccessible over TOR.')
+    logger.info('--enabletor              Setup Basicswap instance to use TOR.')
+    logger.info('--disabletor             Setup Basicswap instance to not use TOR.')
+    logger.info('--usebtcfastsync         Initialise the BTC chain with a snapshot from btcpayserver FastSync.\n'
+                + '                         See https://github.com/btcpayserver/btcpayserver-docker/blob/master/contrib/FastSync/README.md')
 
     logger.info('\n' + 'Known coins: %s', ', '.join(known_coins.keys()))
 
 
 def main():
-    global extract_core_overwrite
     global use_tor_proxy
     data_dir = None
     bin_dir = None
@@ -720,6 +761,8 @@ def main():
     enable_tor = False
     disable_tor = False
     tor_control_password = None
+    use_btc_fastsync = False
+    extract_core_overwrite = True
 
     for v in sys.argv[1:]:
         if len(v) < 2 or v[0] != '-':
@@ -767,6 +810,9 @@ def main():
         if name == 'disabletor':
             disable_tor = True
             continue
+        if name == 'usebtcfastsync':
+            use_btc_fastsync = True
+            continue
         if len(s) == 2:
             if name == 'datadir':
                 data_dir = os.path.expanduser(s[1].strip('"'))
@@ -987,6 +1033,14 @@ def main():
         logger.info('Done.')
         return 0
 
+    extra_opts = {
+        'use_btc_fastsync': use_btc_fastsync,
+        'extract_core_overwrite': extract_core_overwrite,
+        'data_dir': data_dir,
+        'use_containers': use_containers,
+        'tor_control_password': tor_control_password,
+    }
+
     if add_coin != '':
         logger.info('Adding coin: %s', add_coin)
         if not os.path.exists(config_path):
@@ -1010,10 +1064,10 @@ def main():
         settings['use_tor_proxy'] = use_tor_proxy
 
         if not no_cores:
-            prepareCore(add_coin, known_coins[add_coin], settings, data_dir)
+            prepareCore(add_coin, known_coins[add_coin], settings, data_dir, extra_opts)
 
         if not prepare_bin_only:
-            prepareDataDir(add_coin, settings, chain, particl_wallet_mnemonic, use_containers=use_containers)
+            prepareDataDir(add_coin, settings, chain, particl_wallet_mnemonic, extra_opts)
             with open(config_path, 'w') as fp:
                 json.dump(settings, fp, indent=4)
 
@@ -1053,14 +1107,14 @@ def main():
 
     if not no_cores:
         for c in with_coins:
-            prepareCore(c, known_coins[c], settings, data_dir)
+            prepareCore(c, known_coins[c], settings, data_dir, extra_opts)
 
     if prepare_bin_only:
         logger.info('Done.')
         return 0
 
     for c in with_coins:
-        prepareDataDir(c, settings, chain, particl_wallet_mnemonic, use_containers=use_containers, tor_control_password=tor_control_password)
+        prepareDataDir(c, settings, chain, particl_wallet_mnemonic, extra_opts)
 
     with open(config_path, 'w') as fp:
         json.dump(settings, fp, indent=4)
@@ -1116,6 +1170,14 @@ def main():
                 swap_client.setDaemonPID(c, daemons[-1].pid)
                 swap_client.setCoinRunParams(c)
                 swap_client.createCoinInterface(c)
+
+                # Create wallet if it doesn't exist yet
+                if c == Coins.BTC:
+                    swap_client.waitForDaemonRPC(c, with_wallet=False)
+                    wallets = swap_client.callcoinrpc(c, 'listwallets')
+                    if 'wallet.dat' not in wallets:
+                        swap_client.callcoinrpc(c, 'createwallet', ['wallet.dat'])
+
                 swap_client.waitForDaemonRPC(c)
                 swap_client.initialiseWallet(c)
             swap_client.finalise()

doc/install.md

@@ -37,6 +37,12 @@ Adjust `--withcoins` and `--withoutcoins` as desired, eg: `--withcoins=monero,bi
 **Record the mnemonic from the output of the above command.**
 
 
+##### FastSync
+
+Use `--usebtcfastsync` to optionally initialise the Bitcoin datadir with a chain snapshot from btcpayserver FastSync.
+[FastSync README.md](https://github.com/btcpayserver/btcpayserver-docker/blob/master/contrib/FastSync/README.md)
+
+
 #### Set the timezone (optional):
 
 Edit the `.env` file in the docker directory, set TZ to your local timezone.

pgp/sigs/utxo-snapshot-bitcoin-mainnet-720179.tar.asc

@@ -0,0 +1,16 @@
+-----BEGIN PGP SIGNATURE-----
+
+iQIzBAABCAAdFiEEjlF9wS7BzDf2QjqKE/E2UcnPDWsFAmKbqx4ACgkQE/E2UcnP
+DWv5ng//WFFmaiVo+nz8uwtktwEsMsPidGErFY7mkyJxjtviYNZIgxZzuBKvguFO
+wDaeoT90CiWIY0aSkLRopVr7E8oEX3ev6pT7iTsuEeZrJ8HITj5j/B6IwwGvclO7
+Qq1dIZUeNurTFwBXi6TmIPQMtncR9TUWi/20k/NyupsPxlVRBZNrslgKqxKyoXwz
+lcGIV7IqAiUh87/1bdd2qk2K1PF3Z5ah7M2CdjBGRxnJnh0LfZuenFXqdF1EfYq0
+rmXQcYKeHWOcrT/WGpILBgyw7ER6+tqZ8cMD7VdPxX6fsiLiGLA23CCyS54vlnAw
+HN+qOR9KFHaXRkzAn/VEpXoyVEDlDI3qFtqHubBPX4hMuSweQWnx4NcGFe/zXifN
+SyQe5bXwxy0kiggDIczTcrvOCJUwKrYheNp7XHw/920Q1T91wGkk6b/3W2RJWIKx
+jpnMYg+oASe7CFZ2C4pHvlBLOvV1bsPDOuOulti62OtdYAIYjFY2MIR63YyAx+2m
+1SscL1GZfuU01K0XmW9PC1QoEF1tQFjIOI5Am3nivlfuqnfAWXmaKIg/dpvoWr1O
+MQBiQpOrUjHv1fcC1nu3k/20BuzI2o5u99vdUFZzqhg7+VEMXAHnoGwY7+XVPG0A
+VL1vltlOOQzJb0p3Zmt9U7bHE0/sWd/BdhXt4K58GzkgjtZ4aaM=
+=VmAa
+-----END PGP SIGNATURE-----