Merge pull request #890 from cypherstack/testing

Spark mempool scanning and fixes
This commit is contained in:
julian-CStack 2024-06-14 15:57:53 -06:00 committed by GitHub
commit b789f2cbb9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 401 additions and 243 deletions

View file

@ -26,12 +26,20 @@ import '../services/event_bus/events/global/tor_status_changed_event.dart';
import '../services/event_bus/global_event_bus.dart';
import '../services/tor_service.dart';
import '../utilities/amount/amount.dart';
import '../utilities/extensions/impl/string.dart';
import '../utilities/logger.dart';
import '../utilities/prefs.dart';
import '../wallets/crypto_currency/crypto_currency.dart';
import '../wallets/crypto_currency/interfaces/electrumx_currency_interface.dart';
import 'client_manager.dart';
typedef SparkMempoolData = ({
String txid,
List<String> serialContext,
List<String> lTags,
List<String> coins,
});
class WifiOnlyException implements Exception {}
class ElectrumXNode {
@ -1038,10 +1046,9 @@ class ElectrumXClient {
command: "spark.getmempooltxids",
);
// TODO verify once server is live
final txids = List<String>.from(response as List).toSet();
// final map = Map<String, dynamic>.from(response as Map);
// final txids = List<String>.from(map["tags"] as List).toSet();
final txids = List<String>.from(response as List)
.map((e) => e.toHexReversedFromBase64)
.toSet();
Logging.instance.log(
"Finished ElectrumXClient.getMempoolTxids(). "
@ -1057,7 +1064,7 @@ class ElectrumXClient {
}
/// Returns the txids of the current transactions found in the mempool
Future<Map<String, dynamic>> getMempoolSparkData({
Future<List<SparkMempoolData>> getMempoolSparkData({
String? requestID,
required List<String> txids,
}) async {
@ -1066,11 +1073,27 @@ class ElectrumXClient {
final response = await request(
requestID: requestID,
command: "spark.getmempooltxs",
args: txids,
args: [
{
"txids": txids,
},
],
);
// TODO verify once server is live
final map = Map<String, dynamic>.from(response as Map);
final List<SparkMempoolData> result = [];
for (final entry in map.entries) {
result.add(
(
txid: entry.key,
serialContext:
List<String>.from(entry.value["Serial_context"] as List),
// the space after lTags is required lol
lTags: List<String>.from(entry.value["lTags "] as List),
coins: List<String>.from(entry.value["Coins"] as List),
),
);
}
Logging.instance.log(
"Finished ElectrumXClient.getMempoolSparkData(txids: $txids). "
@ -1078,9 +1101,9 @@ class ElectrumXClient {
level: LogLevel.Info,
);
return map;
} catch (e) {
Logging.instance.log(e, level: LogLevel.Error);
return result;
} catch (e, s) {
Logging.instance.log("$e\n$s", level: LogLevel.Error);
rethrow;
}
}

View file

@ -285,17 +285,30 @@ class _Step3ViewState extends ConsumerState<Step3View> {
);
if (response.value == null) {
if (mounted) {
if (context.mounted) {
Navigator.of(context).pop();
// TODO: better errors
String? message;
if (response.exception != null) {
message =
response.exception!.toString();
if (message.startsWith(
"FormatException:",
) &&
message.contains("<html>")) {
message =
"${ref.read(efExchangeProvider).name} server error";
}
}
unawaited(
showDialog<void>(
context: context,
barrierDismissible: true,
builder: (_) => StackDialog(
title: "Failed to create trade",
message: response.exception
?.toString(),
message: message ?? "",
),
),
);

View file

@ -669,21 +669,29 @@ class _TradeDetailsViewState extends ConsumerState<TradeDetailsView> {
mainAxisAlignment: MainAxisAlignment.spaceBetween,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
"${trade.exchangeName} address",
style: STextStyles.itemSubtitle(context),
),
const SizedBox(
height: 4,
),
SelectableText(
trade.payInAddress,
style: STextStyles.itemSubtitle12(context),
),
],
Expanded(
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
"${trade.exchangeName} address",
style: STextStyles.itemSubtitle(context),
),
const SizedBox(
height: 4,
),
Row(
children: [
Flexible(
child: SelectableText(
trade.payInAddress,
style: STextStyles.itemSubtitle12(context),
),
),
],
),
],
),
),
if (isDesktop)
IconCopyButton(
@ -760,9 +768,15 @@ class _TradeDetailsViewState extends ConsumerState<TradeDetailsView> {
const SizedBox(
height: 4,
),
SelectableText(
trade.payInAddress,
style: STextStyles.itemSubtitle12(context),
Row(
children: [
Expanded(
child: SelectableText(
trade.payInAddress,
style: STextStyles.itemSubtitle12(context),
),
),
],
),
const SizedBox(
height: 10,

View file

@ -15,11 +15,9 @@ import 'package:event_bus/event_bus.dart';
import 'package:flutter/material.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:flutter_svg/svg.dart';
import '../../global_settings_view/manage_nodes_views/add_edit_node_view.dart';
import '../../global_settings_view/tor_settings/tor_settings_view.dart';
import '../../sub_widgets/nodes_list.dart';
import 'sub_widgets/confirm_full_rescan.dart';
import 'sub_widgets/rescanning_dialog.dart';
import 'package:tuple/tuple.dart';
import 'package:wakelock/wakelock.dart';
import '../../../../providers/providers.dart';
import '../../../../route_generator.dart';
import '../../../../services/event_bus/events/global/blocks_remaining_event.dart';
@ -53,8 +51,11 @@ import '../../../../widgets/rounded_container.dart';
import '../../../../widgets/rounded_white_container.dart';
import '../../../../widgets/stack_dialog.dart';
import '../../../../widgets/tor_subscription.dart';
import 'package:tuple/tuple.dart';
import 'package:wakelock/wakelock.dart';
import '../../global_settings_view/manage_nodes_views/add_edit_node_view.dart';
import '../../global_settings_view/tor_settings/tor_settings_view.dart';
import '../../sub_widgets/nodes_list.dart';
import 'sub_widgets/confirm_full_rescan.dart';
import 'sub_widgets/rescanning_dialog.dart';
/// [eventBus] should only be set during testing
class WalletNetworkSettingsView extends ConsumerStatefulWidget {

View file

@ -117,13 +117,23 @@ class _StepScaffoldState extends ConsumerState<StepScaffold> {
if (mounted) {
Navigator.of(context).pop();
String? message;
if (response.exception != null) {
message = response.exception!.toString();
// TODO: better errors
if (message.startsWith("FormatException:") &&
message.contains("<html>")) {
message = "${ref.read(efExchangeProvider).name} server error";
}
}
unawaited(
showDialog<void>(
context: context,
barrierDismissible: true,
builder: (_) => SimpleDesktopDialog(
title: "Failed to create trade",
message: response.exception?.toString() ?? "",
message: message ?? "",
),
),
);

View file

@ -14,6 +14,8 @@ import 'package:event_bus/event_bus.dart';
import 'package:flutter/material.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:flutter_svg/svg.dart';
import 'package:tuple/tuple.dart';
import '../../../../pages/settings_views/wallet_settings_view/wallet_network_settings_view/wallet_network_settings_view.dart';
import '../../../../providers/providers.dart';
import '../../../../route_generator.dart';
@ -26,7 +28,6 @@ import '../../../../utilities/text_styles.dart';
import '../../../../utilities/util.dart';
import '../../../../widgets/desktop/desktop_dialog.dart';
import '../../../../widgets/desktop/desktop_dialog_close_button.dart';
import 'package:tuple/tuple.dart';
class NetworkInfoButton extends ConsumerStatefulWidget {
const NetworkInfoButton({
@ -226,7 +227,7 @@ class _NetworkInfoButtonState extends ConsumerState<NetworkInfoButton> {
return [
FadePageRoute(
DesktopDialog(
maxHeight: MediaQuery.of(context).size.height - 64,
maxHeight: null,
maxWidth: 580,
child: Column(
mainAxisSize: MainAxisSize.min,
@ -251,17 +252,21 @@ class _NetworkInfoButtonState extends ConsumerState<NetworkInfoButton> {
],
),
),
Padding(
padding: const EdgeInsets.only(
top: 16,
left: 32,
right: 32,
bottom: 32,
),
child: WalletNetworkSettingsView(
walletId: walletId,
initialSyncStatus: _currentSyncStatus,
initialNodeStatus: _currentNodeStatus,
Flexible(
child: Padding(
padding: const EdgeInsets.only(
top: 16,
left: 32,
right: 32,
bottom: 32,
),
child: SingleChildScrollView(
child: WalletNetworkSettingsView(
walletId: walletId,
initialSyncStatus: _currentSyncStatus,
initialNodeStatus: _currentNodeStatus,
),
),
),
),
],

View file

@ -635,7 +635,9 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
// been marked as isUsed.
// TODO: [prio=med] Could (probably should) throw an exception here if txData.usedSparkCoins is null or empty
if (txData.usedSparkCoins != null && txData.usedSparkCoins!.isNotEmpty) {
await _addOrUpdateSparkCoins(txData.usedSparkCoins!);
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(txData.usedSparkCoins!);
});
}
return await updateSentCachedTxData(txData: txData);
@ -648,7 +650,88 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
}
}
// in mem cache
Set<String> _mempoolTxids = {};
Set<String> _mempoolTxidsChecked = {};
Future<List<SparkCoin>> _refreshSparkCoinsMempoolCheck({
required Set<String> privateKeyHexSet,
required int groupId,
}) async {
final start = DateTime.now();
try {
// update cache
_mempoolTxids = await electrumXClient.getMempoolTxids();
// remove any checked txids that are not in the mempool anymore
_mempoolTxidsChecked = _mempoolTxidsChecked.intersection(_mempoolTxids);
// get all unchecked txids currently in mempool
final txidsToCheck = _mempoolTxids.difference(_mempoolTxidsChecked);
if (txidsToCheck.isEmpty) {
return [];
}
// fetch spark data to scan if we own any unconfirmed spark coins
final sparkDataToCheck = await electrumXClient.getMempoolSparkData(
txids: txidsToCheck.toList(),
);
final Set<String> checkedTxids = {};
final List<List<String>> rawCoins = [];
for (final data in sparkDataToCheck) {
for (int i = 0; i < data.coins.length; i++) {
rawCoins.add([
data.coins[i],
data.txid,
data.serialContext.first,
]);
}
checkedTxids.add(data.txid);
}
final result = <SparkCoin>[];
// if there is new data we try and identify the coins
if (rawCoins.isNotEmpty) {
// run identify off main isolate
final myCoins = await compute(
_identifyCoins,
(
anonymitySetCoins: rawCoins,
groupId: groupId,
privateKeyHexSet: privateKeyHexSet,
walletId: walletId,
isTestNet: cryptoCurrency.network == CryptoCurrencyNetwork.test,
),
);
// add checked txids after identification
_mempoolTxidsChecked.addAll(checkedTxids);
result.addAll(myCoins);
}
return result;
} catch (e) {
Logging.instance.log(
"refreshSparkMempoolData() failed: $e",
level: LogLevel.Error,
);
return [];
} finally {
Logging.instance.log(
"$walletId ${info.name} refreshSparkCoinsMempoolCheck() run "
"duration: ${DateTime.now().difference(start)}",
level: LogLevel.Debug,
);
}
}
Future<void> refreshSparkData() async {
final start = DateTime.now();
try {
// start by checking if any previous sets are missing from db and add the
// missing groupIds to the list if sets to check and update
@ -684,15 +767,210 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
),
]);
await _checkAndUpdateCoins();
// refresh spark balance
await refreshSparkBalance();
// Get cached timestamps per groupId. These timestamps are used to check
// and try to id coins that were added to the spark anon set cache
// after that timestamp.
final groupIdTimestampUTCMap =
info.otherData[WalletInfoKeys.firoSparkCacheSetTimestampCache]
as Map? ??
{};
// iterate through the cache, fetching spark coin data that hasn't been
// processed by this wallet yet
final Map<int, List<List<String>>> rawCoinsBySetId = {};
for (int i = 1; i <= latestGroupId; i++) {
final lastCheckedTimeStampUTC =
groupIdTimestampUTCMap[i.toString()] as int? ?? 0;
final info = await FiroCacheCoordinator.getLatestSetInfoForGroupId(
i,
);
final anonymitySetResult =
await FiroCacheCoordinator.getSetCoinsForGroupId(
i,
newerThanTimeStamp: lastCheckedTimeStampUTC,
);
final coinsRaw = anonymitySetResult
.map(
(e) => [
e.serialized,
e.txHash,
e.context,
],
)
.toList();
if (coinsRaw.isNotEmpty) {
rawCoinsBySetId[i] = coinsRaw;
}
// update last checked timestamp data
groupIdTimestampUTCMap[i.toString()] = max(
lastCheckedTimeStampUTC,
info?.timestampUTC ?? lastCheckedTimeStampUTC,
);
}
// get address(es) to get the private key hex strings required for
// identifying spark coins
final sparkAddresses = await mainDB.isar.addresses
.where()
.walletIdEqualTo(walletId)
.filter()
.typeEqualTo(AddressType.spark)
.findAll();
final root = await getRootHDNode();
final Set<String> privateKeyHexSet = sparkAddresses
.map(
(e) =>
root.derivePath(e.derivationPath!.value).privateKey.data.toHex,
)
.toSet();
// try to identify any coins in the unchecked set data
final List<SparkCoin> newlyIdCoins = [];
for (final groupId in rawCoinsBySetId.keys) {
final myCoins = await compute(
_identifyCoins,
(
anonymitySetCoins: rawCoinsBySetId[groupId]!,
groupId: groupId,
privateKeyHexSet: privateKeyHexSet,
walletId: walletId,
isTestNet: cryptoCurrency.network == CryptoCurrencyNetwork.test,
),
);
newlyIdCoins.addAll(myCoins);
}
// if any were found, add to database
if (newlyIdCoins.isNotEmpty) {
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(newlyIdCoins);
});
}
// finally update the cached timestamps in the database
await info.updateOtherData(
newEntries: {
WalletInfoKeys.firoSparkCacheSetTimestampCache:
groupIdTimestampUTCMap,
},
isar: mainDB.isar,
);
// check for spark coins in mempool
final mempoolMyCoins = await _refreshSparkCoinsMempoolCheck(
privateKeyHexSet: privateKeyHexSet,
groupId: latestGroupId,
);
// if any were found, add to database
if (mempoolMyCoins.isNotEmpty) {
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(mempoolMyCoins);
});
}
// get unused and or unconfirmed coins from db
final coinsToCheck = await mainDB.isar.sparkCoins
.where()
.walletIdEqualToAnyLTagHash(walletId)
.filter()
.heightIsNull()
.or()
.isUsedEqualTo(false)
.findAll();
Set<String>? spentCoinTags;
// only fetch tags from db if we need them to compare against any items
// in coinsToCheck
if (coinsToCheck.isNotEmpty) {
spentCoinTags = await FiroCacheCoordinator.getUsedCoinTags(0);
}
// check and update coins if required
final List<SparkCoin> updatedCoins = [];
for (final coin in coinsToCheck) {
SparkCoin updated = coin;
if (updated.height == null) {
final tx = await electrumXCachedClient.getTransaction(
txHash: updated.txHash,
cryptoCurrency: info.coin,
);
if (tx["height"] is int) {
updated = updated.copyWith(height: tx["height"] as int);
}
}
if (updated.height != null &&
spentCoinTags!.contains(updated.lTagHash)) {
updated = coin.copyWith(isUsed: true);
}
updatedCoins.add(updated);
}
// update in db if any have changed
if (updatedCoins.isNotEmpty) {
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(updatedCoins);
});
}
// used to check if balance is spendable or total
final currentHeight = await chainHeight;
// get all unused coins to update wallet spark balance
final unusedCoins = await mainDB.isar.sparkCoins
.where()
.walletIdEqualToAnyLTagHash(walletId)
.filter()
.isUsedEqualTo(false)
.findAll();
final total = Amount(
rawValue: unusedCoins
.map((e) => e.value)
.fold(BigInt.zero, (prev, e) => prev + e),
fractionDigits: cryptoCurrency.fractionDigits,
);
final spendable = Amount(
rawValue: unusedCoins
.where(
(e) =>
e.height != null &&
e.height! + cryptoCurrency.minConfirms <= currentHeight,
)
.map((e) => e.value)
.fold(BigInt.zero, (prev, e) => prev + e),
fractionDigits: cryptoCurrency.fractionDigits,
);
final sparkBalance = Balance(
total: total,
spendable: spendable,
blockedTotal: Amount(
rawValue: BigInt.zero,
fractionDigits: cryptoCurrency.fractionDigits,
),
pendingSpendable: total - spendable,
);
// finally update balance in db
await info.updateBalanceTertiary(
newBalance: sparkBalance,
isar: mainDB.isar,
);
} catch (e, s) {
Logging.instance.log(
"$runtimeType $walletId ${info.name}: $e\n$s",
level: LogLevel.Error,
);
rethrow;
} finally {
Logging.instance.log(
"${info.name} refreshSparkData() duration:"
" ${DateTime.now().difference(start)}",
level: LogLevel.Debug,
);
}
}
@ -722,49 +1000,6 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
return pairs.toSet();
}
Future<void> refreshSparkBalance() async {
final currentHeight = await chainHeight;
final unusedCoins = await mainDB.isar.sparkCoins
.where()
.walletIdEqualToAnyLTagHash(walletId)
.filter()
.isUsedEqualTo(false)
.findAll();
final total = Amount(
rawValue: unusedCoins
.map((e) => e.value)
.fold(BigInt.zero, (prev, e) => prev + e),
fractionDigits: cryptoCurrency.fractionDigits,
);
final spendable = Amount(
rawValue: unusedCoins
.where(
(e) =>
e.height != null &&
e.height! + cryptoCurrency.minConfirms <= currentHeight,
)
.map((e) => e.value)
.fold(BigInt.zero, (prev, e) => prev + e),
fractionDigits: cryptoCurrency.fractionDigits,
);
final sparkBalance = Balance(
total: total,
spendable: spendable,
blockedTotal: Amount(
rawValue: BigInt.zero,
fractionDigits: cryptoCurrency.fractionDigits,
),
pendingSpendable: total - spendable,
);
await info.updateBalanceTertiary(
newBalance: sparkBalance,
isar: mainDB.isar,
);
}
/// Should only be called within the standard wallet [recover] function due to
/// mutex locking. Otherwise behaviour MAY be undefined.
Future<void> recoverSparkWallet({
@ -777,10 +1012,7 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
}
try {
await _checkAndUpdateCoins();
// refresh spark balance
await refreshSparkBalance();
await refreshSparkData();
} catch (e, s) {
Logging.instance.log(
"$runtimeType $walletId ${info.name}: $e\n$s",
@ -790,115 +1022,6 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
}
}
Future<void> _checkAndUpdateCoins() async {
final sparkAddresses = await mainDB.isar.addresses
.where()
.walletIdEqualTo(walletId)
.filter()
.typeEqualTo(AddressType.spark)
.findAll();
final root = await getRootHDNode();
final Set<String> privateKeyHexSet = sparkAddresses
.map(
(e) => root.derivePath(e.derivationPath!.value).privateKey.data.toHex,
)
.toSet();
final Map<int, List<List<String>>> rawCoinsBySetId = {};
final groupIdTimestampUTCMap =
info.otherData[WalletInfoKeys.firoSparkCacheSetTimestampCache]
as Map? ??
{};
final latestSparkCoinId = await electrumXClient.getSparkLatestCoinId();
for (int i = 1; i <= latestSparkCoinId; i++) {
final lastCheckedTimeStampUTC =
groupIdTimestampUTCMap[i.toString()] as int? ?? 0;
final info = await FiroCacheCoordinator.getLatestSetInfoForGroupId(
i,
);
final anonymitySetResult =
await FiroCacheCoordinator.getSetCoinsForGroupId(
i,
newerThanTimeStamp: lastCheckedTimeStampUTC,
);
final coinsRaw = anonymitySetResult
.map(
(e) => [
e.serialized,
e.txHash,
e.context,
],
)
.toList();
if (coinsRaw.isNotEmpty) {
rawCoinsBySetId[i] = coinsRaw;
}
groupIdTimestampUTCMap[i.toString()] = max(
lastCheckedTimeStampUTC,
info?.timestampUTC ?? lastCheckedTimeStampUTC,
);
}
await info.updateOtherData(
newEntries: {
WalletInfoKeys.firoSparkCacheSetTimestampCache: groupIdTimestampUTCMap,
},
isar: mainDB.isar,
);
final List<SparkCoin> newlyIdCoins = [];
for (final groupId in rawCoinsBySetId.keys) {
final myCoins = await compute(
_identifyCoins,
(
anonymitySetCoins: rawCoinsBySetId[groupId]!,
groupId: groupId,
privateKeyHexSet: privateKeyHexSet,
walletId: walletId,
isTestNet: cryptoCurrency.network == CryptoCurrencyNetwork.test,
),
);
newlyIdCoins.addAll(myCoins);
}
await _checkAndMarkCoinsUsedInDB(coinsNotInDbYet: newlyIdCoins);
}
Future<void> _checkAndMarkCoinsUsedInDB({
List<SparkCoin> coinsNotInDbYet = const [],
}) async {
final List<SparkCoin> coins = await mainDB.isar.sparkCoins
.where()
.walletIdEqualToAnyLTagHash(walletId)
.filter()
.isUsedEqualTo(false)
.findAll();
final List<SparkCoin> coinsToWrite = [];
final spentCoinTags = await FiroCacheCoordinator.getUsedCoinTags(0);
for (final coin in coins) {
if (spentCoinTags.contains(coin.lTagHash)) {
coinsToWrite.add(coin.copyWith(isUsed: true));
}
}
for (final coin in coinsNotInDbYet) {
if (spentCoinTags.contains(coin.lTagHash)) {
coinsToWrite.add(coin.copyWith(isUsed: true));
} else {
coinsToWrite.add(coin);
}
}
// update wallet spark coins in isar
await _addOrUpdateSparkCoins(coinsToWrite);
}
// modelled on CSparkWallet::CreateSparkMintTransactions https://github.com/firoorg/firo/blob/39c41e5e7ec634ced3700fe3f4f5509dc2e480d0/src/spark/sparkwallet.cpp#L752
Future<List<TxData>> _createSparkMintTransactions({
required List<UTXO> availableUtxos,
@ -1698,37 +1821,6 @@ mixin SparkInterface<T extends ElectrumXCurrencyInterface>
// ====================== Private ============================================
Future<void> _addOrUpdateSparkCoins(List<SparkCoin> coins) async {
if (coins.isNotEmpty) {
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(coins);
});
}
// update wallet spark coin height
final coinsToCheck = await mainDB.isar.sparkCoins
.where()
.walletIdEqualToAnyLTagHash(walletId)
.filter()
.heightIsNull()
.findAll();
final List<SparkCoin> updatedCoins = [];
for (final coin in coinsToCheck) {
final tx = await electrumXCachedClient.getTransaction(
txHash: coin.txHash,
cryptoCurrency: info.coin,
);
if (tx["height"] is int) {
updatedCoins.add(coin.copyWith(height: tx["height"] as int));
}
}
if (updatedCoins.isNotEmpty) {
await mainDB.isar.writeTxn(() async {
await mainDB.isar.sparkCoins.putAll(updatedCoins);
});
}
}
btc.NetworkType get _bitcoinDartNetwork => btc.NetworkType(
messagePrefix: cryptoCurrency.networkParams.messagePrefix,
bech32: cryptoCurrency.networkParams.bech32Hrp,