Fix BTC witness size estimate.

basicswap/db.py

@@ -295,8 +295,8 @@ class XmrOffer(Base):
     swap_id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
     offer_id = sa.Column(sa.LargeBinary, sa.ForeignKey('offers.offer_id'))
 
-    a_fee_rate = sa.Column(sa.BigInteger)
+    a_fee_rate = sa.Column(sa.BigInteger)  # Chain a fee rate
-    b_fee_rate = sa.Column(sa.BigInteger)
+    b_fee_rate = sa.Column(sa.BigInteger)  # Chain b fee rate
 
     lock_time_1 = sa.Column(sa.Integer)  # Delay before the chain a lock refund tx can be mined
     lock_time_2 = sa.Column(sa.Integer)  # Delay before the follower can spend from the chain a lock refund tx

basicswap/interface/btc.py

@@ -351,7 +351,7 @@ class BTCInterface(CoinInterface):
         if self.sc._restrict_unknown_seed_wallets:
             ensure(addr_info['hdseedid'] == self._expect_seedid_hex, 'unexpected seedid')
 
-    def get_fee_rate(self, conf_target=2):
+    def get_fee_rate(self, conf_target: int = 2):
         try:
             fee_rate = self.rpc_callback('estimatesmartfee', [conf_target])['feerate']
             assert (fee_rate > 0.0), 'Non positive feerate'
@@ -410,8 +410,8 @@ class BTCInterface(CoinInterface):
         assert (len(pk) == 33)
         return self.pkh_to_address(hash160(pk))
 
-    def getNewSecretKey(self):
+    def getNewSecretKey(self) -> bytes:
-        return getSecretInt()
+        return i2b(getSecretInt())
 
     def getPubkey(self, privkey):
         return PublicKey.from_secret(privkey).format()
@@ -486,7 +486,7 @@ class BTCInterface(CoinInterface):
     def fundSCLockTx(self, tx_bytes, feerate, vkbv=None):
         return self.fundTx(tx_bytes, feerate)
 
-    def extractScriptLockRefundScriptValues(self, script_bytes):
+    def extractScriptLockRefundScriptValues(self, script_bytes: bytes):
         script_len = len(script_bytes)
         ensure(script_len > 73, 'Bad script length')
         ensure_op(script_bytes[0] == OP_IF)
@@ -517,7 +517,7 @@ class BTCInterface(CoinInterface):
 
         return pk1, pk2, csv_val, pk3
 
-    def genScriptLockRefundTxScript(self, Kal, Kaf, csv_val):
+    def genScriptLockRefundTxScript(self, Kal, Kaf, csv_val) -> CScript:
 
         Kal_enc = Kal if len(Kal) == 33 else self.encodePubkey(Kal)
         Kaf_enc = Kaf if len(Kaf) == 33 else self.encodePubkey(Kaf)
@@ -553,7 +553,7 @@ class BTCInterface(CoinInterface):
         dummy_witness_stack = self.getScriptLockTxDummyWitness(script_lock)
         witness_bytes = self.getWitnessStackSerialisedLength(dummy_witness_stack)
         vsize = self.getTxVSize(tx, add_witness_bytes=witness_bytes)
-        pay_fee = int(tx_fee_rate * vsize // 1000)
+        pay_fee = round(tx_fee_rate * vsize / 1000)
         tx.vout[0].nValue = locked_coin - pay_fee
 
         tx.rehash()
@@ -588,7 +588,7 @@ class BTCInterface(CoinInterface):
         dummy_witness_stack = self.getScriptLockRefundSpendTxDummyWitness(script_lock_refund)
         witness_bytes = self.getWitnessStackSerialisedLength(dummy_witness_stack)
         vsize = self.getTxVSize(tx, add_witness_bytes=witness_bytes)
-        pay_fee = int(tx_fee_rate * vsize // 1000)
+        pay_fee = round(tx_fee_rate * vsize / 1000)
         tx.vout[0].nValue = locked_coin - pay_fee
 
         tx.rehash()
@@ -624,7 +624,7 @@ class BTCInterface(CoinInterface):
         dummy_witness_stack = self.getScriptLockRefundSwipeTxDummyWitness(script_lock_refund)
         witness_bytes = self.getWitnessStackSerialisedLength(dummy_witness_stack)
         vsize = self.getTxVSize(tx, add_witness_bytes=witness_bytes)
-        pay_fee = int(tx_fee_rate * vsize // 1000)
+        pay_fee = round(tx_fee_rate * vsize / 1000)
         tx.vout[0].nValue = locked_coin - pay_fee
 
         tx.rehash()
@@ -633,7 +633,7 @@ class BTCInterface(CoinInterface):
 
         return tx.serialize()
 
-    def createSCLockSpendTx(self, tx_lock_bytes, script_lock, pkh_dest, tx_fee_rate, vkbv=None):
+    def createSCLockSpendTx(self, tx_lock_bytes, script_lock, pkh_dest, tx_fee_rate, vkbv=None, fee_info={}):
         tx_lock = self.loadTx(tx_lock_bytes)
         output_script = self.getScriptDest(script_lock)
         locked_n = findOutput(tx_lock, output_script)
@@ -653,9 +653,14 @@ class BTCInterface(CoinInterface):
         dummy_witness_stack = self.getScriptLockTxDummyWitness(script_lock)
         witness_bytes = self.getWitnessStackSerialisedLength(dummy_witness_stack)
         vsize = self.getTxVSize(tx, add_witness_bytes=witness_bytes)
-        pay_fee = int(tx_fee_rate * vsize // 1000)
+        pay_fee = round(tx_fee_rate * vsize / 1000)
         tx.vout[0].nValue = locked_coin - pay_fee
 
+        fee_info['fee_paid'] = pay_fee
+        fee_info['rate_used'] = tx_fee_rate
+        fee_info['witness_bytes'] = witness_bytes
+        fee_info['vsize'] = vsize
+
         tx.rehash()
         self._log.info('createSCLockSpendTx %s:\n    fee_rate, vsize, fee: %ld, %ld, %ld.',
                        i2h(tx.sha256), tx_fee_rate, vsize, pay_fee)
@@ -1070,7 +1075,7 @@ class BTCInterface(CoinInterface):
     def getBLockSpendTxFee(self, tx, fee_rate: int) -> int:
         witness_bytes = 109
         vsize = self.getTxVSize(tx, add_witness_bytes=witness_bytes)
-        pay_fee = int(fee_rate * vsize // 1000)
+        pay_fee = round(fee_rate * vsize / 1000)
         self._log.info(f'BLockSpendTx  fee_rate, vsize, fee: {fee_rate}, {vsize}, {pay_fee}.')
         return pay_fee
 
@@ -1244,37 +1249,37 @@ class BTCInterface(CoinInterface):
         # Only one prevout exists
         return 0
 
-    def getScriptLockTxDummyWitness(self, script):
+    def getScriptLockTxDummyWitness(self, script: bytes):
         return [
-            b''.hex(),
+            b'',
-            bytes(72).hex(),
+            bytes(72),
-            bytes(72).hex(),
+            bytes(72),
-            bytes(len(script)).hex()
+            bytes(len(script))
         ]
 
-    def getScriptLockRefundSpendTxDummyWitness(self, script):
+    def getScriptLockRefundSpendTxDummyWitness(self, script: bytes):
         return [
-            b''.hex(),
+            b'',
-            bytes(72).hex(),
+            bytes(72),
-            bytes(72).hex(),
+            bytes(72),
-            bytes((1,)).hex(),
+            bytes((1,)),
-            bytes(len(script)).hex()
+            bytes(len(script))
         ]
 
-    def getScriptLockRefundSwipeTxDummyWitness(self, script):
+    def getScriptLockRefundSwipeTxDummyWitness(self, script: bytes):
         return [
-            bytes(72).hex(),
+            bytes(72),
-            b''.hex(),
+            b'',
-            bytes(len(script)).hex()
+            bytes(len(script))
         ]
 
     def getWitnessStackSerialisedLength(self, witness_stack):
         length = getCompactSizeLen(len(witness_stack))
         for e in witness_stack:
-            length += getWitnessElementLen(len(e) // 2)  # hex -> bytes
+            length += getWitnessElementLen(len(e))
 
         # See core SerializeTransaction
-        length += 32 + 4 + 1 + 4  # vinDummy
+        length += 1  # vinDummy
         length += 1  # flags
         return length
 

basicswap/interface/dash.py

@@ -65,7 +65,7 @@ class DASHInterface(BTCInterface):
     def getBLockSpendTxFee(self, tx, fee_rate: int) -> int:
         add_bytes = 107
         size = len(tx.serialize_with_witness()) + add_bytes
-        pay_fee = int(fee_rate * size // 1000)
+        pay_fee = round(fee_rate * size / 1000)
         self._log.info(f'BLockSpendTx  fee_rate, size, fee: {fee_rate}, {size}, {pay_fee}.')
         return pay_fee
 

basicswap/interface/firo.py

@@ -187,7 +187,7 @@ class FIROInterface(BTCInterface):
     def getBLockSpendTxFee(self, tx, fee_rate: int) -> int:
         add_bytes = 107
         size = len(tx.serialize_with_witness()) + add_bytes
-        pay_fee = int(fee_rate * size // 1000)
+        pay_fee = round(fee_rate * size / 1000)
         self._log.info(f'BLockSpendTx  fee_rate, size, fee: {fee_rate}, {size}, {pay_fee}.')
         return pay_fee
 

basicswap/interface/part.py

@@ -17,7 +17,6 @@ from basicswap.contrib.test_framework.script import (
     OP_DUP, OP_HASH160, OP_EQUALVERIFY, OP_CHECKSIG
 )
 from basicswap.util import (
-    i2b,
     ensure,
     make_int,
     TemporaryError,
@@ -113,7 +112,7 @@ class PARTInterface(BTCInterface):
     def getWitnessStackSerialisedLength(self, witness_stack):
         length = getCompactSizeLen(len(witness_stack))
         for e in witness_stack:
-            length += getWitnessElementLen(len(e) // 2)  # hex -> bytes
+            length += getWitnessElementLen(len(e))
         return length
 
     def getWalletRestoreHeight(self) -> int:
@@ -169,7 +168,7 @@ class PARTInterfaceBlind(PARTInterface):
     def createSCLockTx(self, value: int, script: bytearray, vkbv) -> bytes:
 
         # Nonce is derived from vkbv, ephemeral_key isn't used
-        ephemeral_key = i2b(self.getNewSecretKey())
+        ephemeral_key = self.getNewSecretKey()
         ephemeral_pubkey = self.getPubkey(ephemeral_key)
         assert (len(ephemeral_pubkey) == 33)
         nonce = self.getScriptLockTxNonce(vkbv)
@@ -208,7 +207,7 @@ class PARTInterfaceBlind(PARTInterface):
         lock_tx_obj = self.rpc_callback('decoderawtransaction', [tx_lock_bytes.hex()])
         assert (self.getTxid(tx_lock_bytes).hex() == lock_tx_obj['txid'])
         # Nonce is derived from vkbv, ephemeral_key isn't used
-        ephemeral_key = i2b(self.getNewSecretKey())
+        ephemeral_key = self.getNewSecretKey()
         ephemeral_pubkey = self.getPubkey(ephemeral_key)
         assert (len(ephemeral_pubkey) == 33)
         nonce = self.getScriptLockTxNonce(vkbv)
@@ -231,9 +230,10 @@ class PARTInterfaceBlind(PARTInterface):
 
         # Set dummy witness data for fee estimation
         dummy_witness_stack = self.getScriptLockTxDummyWitness(script_lock)
+        dummy_witness_stack = [x.hex() for x in dummy_witness_stack]
 
         # Use a junk change pubkey to avoid adding unused keys to the wallet
-        zero_change_key = i2b(self.getNewSecretKey())
+        zero_change_key = self.getNewSecretKey()
         zero_change_pubkey = self.getPubkey(zero_change_key)
         inputs_info = {'0': {'value': input_blinded_info['amount'], 'blind': input_blinded_info['blind'], 'witnessstack': dummy_witness_stack}}
         outputs_info = rv['amounts']
@@ -279,9 +279,10 @@ class PARTInterfaceBlind(PARTInterface):
 
         # Set dummy witness data for fee estimation
         dummy_witness_stack = self.getScriptLockRefundSpendTxDummyWitness(script_lock_refund)
+        dummy_witness_stack = [x.hex() for x in dummy_witness_stack]
 
         # Use a junk change pubkey to avoid adding unused keys to the wallet
-        zero_change_key = i2b(self.getNewSecretKey())
+        zero_change_key = self.getNewSecretKey()
         zero_change_pubkey = self.getPubkey(zero_change_key)
         inputs_info = {'0': {'value': input_blinded_info['amount'], 'blind': input_blinded_info['blind'], 'witnessstack': dummy_witness_stack}}
         outputs_info = rv['amounts']
@@ -483,9 +484,9 @@ class PARTInterfaceBlind(PARTInterface):
         dummy_witness_stack = self.getScriptLockTxDummyWitness(script_lock)
 
         # Use a junk change pubkey to avoid adding unused keys to the wallet
-        zero_change_key = i2b(self.getNewSecretKey())
+        zero_change_key = self.getNewSecretKey()
         zero_change_pubkey = self.getPubkey(zero_change_key)
-        inputs_info = {'0': {'value': blinded_info['amount'], 'blind': blinded_info['blind'], 'witnessstack': dummy_witness_stack}}
+        inputs_info = {'0': {'value': blinded_info['amount'], 'blind': blinded_info['blind'], 'witnessstack': [x.hex() for x in dummy_witness_stack]}}
         outputs_info = rv['amounts']
         options = {
             'changepubkey': zero_change_pubkey.hex(),
@@ -605,9 +606,10 @@ class PARTInterfaceBlind(PARTInterface):
 
         # Set dummy witness data for fee estimation
         dummy_witness_stack = self.getScriptLockRefundSwipeTxDummyWitness(script_lock_refund)
+        dummy_witness_stack = [x.hex() for x in dummy_witness_stack]
 
         # Use a junk change pubkey to avoid adding unused keys to the wallet
-        zero_change_key = i2b(self.getNewSecretKey())
+        zero_change_key = self.getNewSecretKey()
         zero_change_pubkey = self.getPubkey(zero_change_key)
         inputs_info = {'0': {'value': input_blinded_info['amount'], 'blind': input_blinded_info['blind'], 'witnessstack': dummy_witness_stack}}
         outputs_info = rv['amounts']

basicswap/interface/pivx.py

@@ -95,7 +95,7 @@ class PIVXInterface(BTCInterface):
     def getBLockSpendTxFee(self, tx, fee_rate: int) -> int:
         add_bytes = 107
         size = len(tx.serialize_with_witness()) + add_bytes
-        pay_fee = int(fee_rate * size // 1000)
+        pay_fee = round(fee_rate * size / 1000)
         self._log.info(f'BLockSpendTx  fee_rate, size, fee: {fee_rate}, {size}, {pay_fee}.')
         return pay_fee
 

basicswap/interface/xmr.py

@@ -27,6 +27,7 @@ from coincurve.dleag import (
 from basicswap.interface import (
     Curves)
 from basicswap.util import (
+    i2b,
     dumpj,
     ensure,
     make_int,
@@ -206,12 +207,13 @@ class XMRInterface(CoinInterface):
             self.openWallet(self._wallet_filename)
             return self.rpc_wallet_cb('create_address', {'account_index': 0})['address']
 
-    def get_fee_rate(self, conf_target=2):
+    def get_fee_rate(self, conf_target: int = 2):
         self._log.warning('TODO - estimate fee rate?')
         return 0.0, 'unused'
 
     def getNewSecretKey(self) -> bytes:
-        return edu.get_secret()
+        # Note: Returned bytes are in big endian order
+        return i2b(edu.get_secret())
 
     def pubkey(self, key: bytes) -> bytes:
         return edf.scalarmult_B(key)

basicswap/ui/util.py

@@ -366,7 +366,7 @@ def describeBid(swap_client, bid, xmr_swap, offer, xmr_offer, bid_events, edit_b
             if xmr_swap:
                 if view_tx_id == xmr_swap.a_lock_tx_id and xmr_swap.a_lock_tx:
                     data['view_tx_hex'] = xmr_swap.a_lock_tx.hex()
-                    data['chain_a_lock_tx_inputs'] = ci_from.listInputs(xmr_swap.a_lock_tx)
+                    data['chain_a_lock_tx_inputs'] = ci_leader.listInputs(xmr_swap.a_lock_tx)
                 if view_tx_id == xmr_swap.a_lock_refund_tx_id and xmr_swap.a_lock_refund_tx:
                     data['view_tx_hex'] = xmr_swap.a_lock_refund_tx.hex()
                 if view_tx_id == xmr_swap.a_lock_refund_spend_tx_id and xmr_swap.a_lock_refund_spend_tx:
@@ -375,7 +375,7 @@ def describeBid(swap_client, bid, xmr_swap, offer, xmr_offer, bid_events, edit_b
                     data['view_tx_hex'] = xmr_swap.a_lock_spend_tx.hex()
 
                 if 'view_tx_hex' in data:
-                    data['view_tx_desc'] = json.dumps(ci_from.describeTx(data['view_tx_hex']), indent=4)
+                    data['view_tx_desc'] = json.dumps(ci_leader.describeTx(data['view_tx_hex']), indent=4)
     else:
         if offer.lock_type == TxLockTypes.SEQUENCE_LOCK_TIME:
             if bid.initiate_tx and bid.initiate_tx.block_time is not None:

basicswap/util/__init__.py

@@ -144,7 +144,7 @@ def make_int(v, scale=8, r=0) -> int:  # r = 0, no rounding, fail, r > 0 round u
     return rv * sign
 
 
-def validate_amount(amount, scale=8) -> bool:
+def validate_amount(amount, scale: int = 8) -> bool:
     str_amount = float_to_str(amount) if type(amount) == float else str(amount)
     has_decimal = False
     for c in str_amount:
@@ -160,7 +160,7 @@ def validate_amount(amount, scale=8) -> bool:
     return True
 
 
-def format_amount(i, display_scale, scale=None):
+def format_amount(i: int, display_scale: int, scale: int = None) -> str:
     if not isinstance(i, int):
         raise ValueError('Amount must be an integer.')  # Raise error instead of converting as amounts should always be integers
     if scale is None:

doc/release-notes.md

@@ -6,6 +6,7 @@
   - Runs the adaptor-sig protocol with leader and follower swapped to
     enable offers from no-script coins to script coins.
 - smsg: Outbox messages are removed when expired.
+- Fixed BTC witness size estimation.
 
 
 0.0.63

tests/basicswap/test_btc_xmr.py

@@ -548,6 +548,85 @@ class BasicSwapTest(TestFunctions):
         rv = read_json_api(1800, 'getcoinseed', {'coin': 'XMR'})
         assert (rv['address'] == '47H7UDLzYEsR28BWttxp59SP1UVSxs4VKDJYSfmz7Wd4Fue5VWuoV9x9eejunwzVSmHWN37gBkaAPNf9VD4bTvwQKsBVWyK')
 
+    def test_010_txn_size(self):
+        logging.info('---------- Test {} txn_size'.format(self.test_coin_from.name))
+
+        swap_clients = self.swap_clients
+        ci = swap_clients[0].ci(self.test_coin_from)
+        pi = swap_clients[0].pi(SwapTypes.XMR_SWAP)
+
+        amount: int = ci.make_int(random.uniform(0.1, 2.0), r=1)
+
+        # Record unspents before createSCLockTx as the used ones will be locked
+        unspents = self.callnoderpc('listunspent')
+
+        # fee_rate is in sats/kvB
+        fee_rate: int = 1000
+
+        a = ci.getNewSecretKey()
+        b = ci.getNewSecretKey()
+
+        A = ci.getPubkey(a)
+        B = ci.getPubkey(b)
+        lock_tx_script = pi.genScriptLockTxScript(ci, A, B)
+
+        lock_tx = ci.createSCLockTx(amount, lock_tx_script)
+        lock_tx = ci.fundSCLockTx(lock_tx, fee_rate)
+        lock_tx = ci.signTxWithWallet(lock_tx)
+
+        unspents_after = self.callnoderpc('listunspent')
+        assert (len(unspents) > len(unspents_after))
+
+        tx_decoded = self.callnoderpc('decoderawtransaction', [lock_tx.hex()])
+        txid = tx_decoded['txid']
+
+        vsize = tx_decoded['vsize']
+        expect_fee_int = round(fee_rate * vsize / 1000)
+        expect_fee = ci.format_amount(expect_fee_int)
+
+        out_value: int = 0
+        for txo in tx_decoded['vout']:
+            if 'value' in txo:
+                out_value += ci.make_int(txo['value'])
+        in_value: int = 0
+        for txi in tx_decoded['vin']:
+            for utxo in unspents:
+                if 'vout' not in utxo:
+                    continue
+                if utxo['txid'] == txi['txid'] and utxo['vout'] == txi['vout']:
+                    in_value += ci.make_int(utxo['amount'])
+                    break
+        fee_value = in_value - out_value
+
+        self.callnoderpc('sendrawtransaction', [lock_tx.hex()])
+        rv = self.callnoderpc('gettransaction', [txid])
+        wallet_tx_fee = -ci.make_int(rv['fee'])
+
+        assert (wallet_tx_fee == fee_value)
+        assert (wallet_tx_fee == expect_fee_int)
+
+        addr_out = ci.getNewAddress(True)
+        pkh_out = ci.decodeAddress(addr_out)
+        fee_info = {}
+        lock_spend_tx = ci.createSCLockSpendTx(lock_tx, lock_tx_script, pkh_out, fee_rate, fee_info=fee_info)
+        vsize_estimated: int = fee_info['vsize']
+
+        tx_decoded = self.callnoderpc('decoderawtransaction', [lock_spend_tx.hex()])
+        txid = tx_decoded['txid']
+
+        witness_stack = [
+            b'',
+            ci.signTx(a, lock_spend_tx, 0, lock_tx_script, amount),
+            ci.signTx(b, lock_spend_tx, 0, lock_tx_script, amount),
+            lock_tx_script,
+        ]
+        lock_spend_tx = ci.setTxSignature(lock_spend_tx, witness_stack)
+        tx_decoded = self.callnoderpc('decoderawtransaction', [lock_spend_tx.hex()])
+        vsize_actual: int = tx_decoded['vsize']
+
+        assert (vsize_actual <= vsize_estimated and vsize_estimated - vsize_actual < 4)
+        assert (self.callnoderpc('sendrawtransaction', [lock_spend_tx.hex()]) == txid)
+
     def test_01_a_full_swap(self):
         if not self.has_segwit:
             return

tests/basicswap/test_other.py

@@ -169,8 +169,8 @@ class Test(unittest.TestCase):
         coin_settings = {'rpcport': 0, 'rpcauth': 'none'}
         coin_settings.update(self.REQUIRED_SETTINGS)
         ci = BTCInterface(coin_settings, 'regtest')
-        vk_sign = i2b(ci.getNewSecretKey())
+        vk_sign = ci.getNewSecretKey()
-        vk_encrypt = i2b(ci.getNewSecretKey())
+        vk_encrypt = ci.getNewSecretKey()
 
         pk_sign = ci.getPubkey(vk_sign)
         pk_encrypt = ci.getPubkey(vk_encrypt)
@@ -193,7 +193,7 @@ class Test(unittest.TestCase):
         coin_settings.update(self.REQUIRED_SETTINGS)
         ci = BTCInterface(coin_settings, 'regtest')
 
-        vk = i2b(ci.getNewSecretKey())
+        vk = ci.getNewSecretKey()
         pk = ci.getPubkey(vk)
 
         message = 'test signing message'
@@ -208,7 +208,7 @@ class Test(unittest.TestCase):
         coin_settings.update(self.REQUIRED_SETTINGS)
         ci = BTCInterface(coin_settings, 'regtest')
 
-        vk = i2b(ci.getNewSecretKey())
+        vk = ci.getNewSecretKey()
         pk = ci.getPubkey(vk)
         sig = ci.signCompact(vk, 'test signing message')
         assert (len(sig) == 64)
@@ -223,7 +223,7 @@ class Test(unittest.TestCase):
         coin_settings.update(self.REQUIRED_SETTINGS)
         ci = BTCInterface(coin_settings, 'regtest')
 
-        vk = i2b(ci.getNewSecretKey())
+        vk = ci.getNewSecretKey()
         pk = ci.getPubkey(vk)
         sig = ci.signRecoverable(vk, 'test signing message')
         assert (len(sig) == 65)
@@ -248,7 +248,7 @@ class Test(unittest.TestCase):
 
         ci = XMRInterface(coin_settings, 'regtest')
 
-        key = i2b(ci.getNewSecretKey())
+        key = ci.getNewSecretKey()
         proof = ci.proveDLEAG(key)
         assert (ci.verifyDLEAG(proof))