grin: regression: fast sync from scratch freeze after txhashset validated

Threads in contention appear to be as follows:

Process 20906 stopped
* thread #1: tid = 20906, 0x00007f631d053e00 libpthread.so.0`__GI___nanosleep + 64, name = 'grin', stop reason = signal SIGSTOP
  thread #2: tid = 20908, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'grin'
  thread #3: tid = 20909, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'grin'
  thread #4: tid = 20910, 0x00007f631d053e00 libpthread.so.0`__GI___nanosleep + 64, name = 'seed'
  thread #5: tid = 20911, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'sync'
  thread #6: tid = 20912, 0x00007f631d053e00 libpthread.so.0`__GI___nanosleep + 64, name = 'p2p-server'
  thread #7: tid = 20913, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'p2p-monitor'
  thread #8: tid = 20914, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'apis'
  thread #9: tid = 20915, 0x00007f631d053e00 libpthread.so.0`__GI___nanosleep + 64, name = 'dandelion'
  thread #10: tid = 20916, 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086, name = 'ui'
  thread #11: tid = 20917, 0x00007f631cb6785f libc.so.6`epoll_wait + 79, name = 'apis'
  thread #12: tid = 20918, 0x00007f631cb6785f libc.so.6`epoll_wait + 79, name = 'apis'
  thread #13: tid = 20919, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'tokio-runtime-w'
  thread #14: tid = 20920, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'tokio-runtime-w'
  thread #15: tid = 20921, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'ui'
  thread #16: tid = 20922, 0x00007f631d04fffc libpthread.so.0`__pthread_cond_wait + 508, name = 'ui'
  thread #17: tid = 20930, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #18: tid = 20931, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #19: tid = 20932, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #20: tid = 20933, 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086, name = 'peer'
  thread #21: tid = 20934, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #22: tid = 20935, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #23: tid = 20986, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #24: tid = 20987, 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086, name = 'peer'
  thread #25: tid = 20988, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #26: tid = 20989, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
  thread #27: tid = 21019, 0x00007f631d05329c libpthread.so.0`__lll_lock_wait + 28, name = 'peer'
(lldb) thread backtrace 10
  thread #10, name = 'ui'
    frame #0: 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086
    frame #1: 0x0000555d8e703bbb grin`mdb_txn_renew0(txn=0x00007f631b74f000) at mdb.c:2692
    frame #2: 0x0000555d8e704d4c grin`mdb_txn_begin(env=0x00007f631c215300, parent=0x0000000000000000, flags=0, ret=0x00007ee318ffc7b0) at mdb.c:2850
    frame #3: 0x0000555d8e2b24b6 grin`lmdb_zero::tx::ConstTransaction::new::h159de4c669252980(env=Arc<lmdb_zero::env::Environment> @ 0x00007ee318ffc788, parent=Option<&mut lmdb_zero::tx::ConstTransaction> @ 0x00007ee318ffc790, flags=0) at tx.rs:514
    frame #4: 0x0000555d8e2b26ec grin`lmdb_zero::tx::WriteTransaction::new::h9581ef9a44edf59e(env=Arc<lmdb_zero::env::Environment> @ 0x00007ee318ffc868) at tx.rs:870
    frame #5: 0x0000555d8e2b719b grin`grin_store::lmdb::Store::batch::h02fe585ae6896745(self=0x00007f631c222550) at lmdb.rs:156
    frame #6: 0x0000555d8e1dcaeb grin`grin_chain::store::ChainStore::batch::h29ee00c7c403e5ac(self=0x00007f631c222550) at store.rs:155
    frame #7: 0x0000555d8e21c8e7 grin`grin_chain::chain::Chain::difficulty_iter::h6477bea2a8aa6fd2(self=0x00007f631c21b090) at chain.rs:967
    frame #8: 0x0000555d8d9f9b0f grin`grin_servers::grin::server::Server::get_server_stats::h0af9e1701e8d7b62(self=0x00007f631c22ac10) at server.rs:399
    frame #9: 0x0000555d8d6cc613 grin`grin::tui::ui::Controller::run::h97276a1748ed62b3(self=0x00007ee318ffe458, server=Arc<grin_servers::grin::server::Server> @ 0x00007ee318ffdc90, running=Arc<core::sync::atomic::AtomicBool> @ 0x00007ee318ffdc98) at ui.rs:194
    frame #10: 0x0000555d8d82179e grin`grin::cmd::server::start_server_tui::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::hdea824521f735f39 at server.rs:59
    frame #11: 0x0000555d8d8531ab grin`std::sys_common::backtrace::__rust_begin_short_backtrace::h0d72fd4ba4520e2a(f=closure @ 0x00007ee318ffe790) at backtrace.rs:136
    frame #12: 0x0000555d8d69d74d grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::hcd320a8dafda16f0 at mod.rs:409
    frame #13: 0x0000555d8d7be8dd grin`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::hb7d5f821c142485f(self=AssertUnwindSafe<closure> @ 0x00007ee318ffe7d8, _args=<unavailable>) at panic.rs:308
    frame #14: 0x0000555d8d73b800 grin`std::panicking::try::do_call::h2833e337b5306124(data="") at panicking.rs:310
    frame #15: 0x0000555d8ebd832a grin`__rust_maybe_catch_panic at lib.rs:105
    frame #16: 0x0000555d8d73b658 grin`std::panicking::try::h4b9632071408fa5e(f=AssertUnwindSafe<closure> @ 0x00007ee318ffe8a0) at panicking.rs:289
    frame #17: 0x0000555d8d7be90d grin`std::panic::catch_unwind::h3ecf08ceb6430c4b(f=AssertUnwindSafe<closure> @ 0x00007ee318ffe928) at panic.rs:392
    frame #18: 0x0000555d8d69d27c grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::h96f64c6aac937800 at mod.rs:408
    frame #19: 0x0000555d8d69d7f7 grin`_$LT$F$u20$as$u20$alloc..boxed..FnBox$LT$A$GT$$GT$::call_box::hcd9d09f3178345ef(self=0x00007f631c220660, args=<unavailable>) at boxed.rs:640
    frame #20: 0x0000555d8ebb02eb grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 [inlined] _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h75e539106a648d39 at boxed.rs:650
    frame #21: 0x0000555d8ebb02e8 grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 at thread.rs:24
    frame #22: 0x0000555d8ebb3a76 grin`std::sys::unix::thread::Thread::new::thread_start::h7d7a420a78cfa84d at thread.rs:90
    frame #23: 0x00007f631d04a075 libpthread.so.0`start_thread + 229
    frame #24: 0x00007f631cb6753f libc.so.6`__GI___clone + 63

(lldb) thread backtrace 20
  thread #20, name = 'peer'
    frame #0: 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086
    frame #1: 0x0000555d8e703bbb grin`mdb_txn_renew0(txn=0x00007f631b74f000) at mdb.c:2692
    frame #2: 0x0000555d8e704d4c grin`mdb_txn_begin(env=0x00007f631c215300, parent=0x0000000000000000, flags=0, ret=0x00007ee314fedf40) at mdb.c:2850
    frame #3: 0x0000555d8e2b24b6 grin`lmdb_zero::tx::ConstTransaction::new::h159de4c669252980(env=Arc<lmdb_zero::env::Environment> @ 0x00007ee314fedf18, parent=Option<&mut lmdb_zero::tx::ConstTransaction> @ 0x00007ee314fedf20, flags=0) at tx.rs:514
    frame #4: 0x0000555d8e2b26ec grin`lmdb_zero::tx::WriteTransaction::new::h9581ef9a44edf59e(env=Arc<lmdb_zero::env::Environment> @ 0x00007ee314fedff8) at tx.rs:870
    frame #5: 0x0000555d8e2b719b grin`grin_store::lmdb::Store::batch::h02fe585ae6896745(self=0x00007f631c222550) at lmdb.rs:156
    frame #6: 0x0000555d8e1dcaeb grin`grin_chain::store::ChainStore::batch::h29ee00c7c403e5ac(self=0x00007f631c222550) at store.rs:155
    frame #7: 0x0000555d8e20e7ca grin`grin_chain::chain::Chain::process_block_no_orphans::h90c33d00b8b6eb51(self=0x00007f631c21b090, b=Block @ 0x00007ee314ff0c48, opts=(bits = 2)) at chain.rs:238
    frame #8: 0x0000555d8e20e300 grin`grin_chain::chain::Chain::process_block::hb4d8b251997d216b(self=0x00007f631c21b090, b=<unavailable>, opts=(bits = 2)) at chain.rs:218
    frame #9: 0x0000555d8da3c396 grin`grin_servers::common::adapters::NetToChainAdapter::process_block::h9643770dd45293b1(self=0x00007f631c22ae10, b=Block @ 0x00007ee314ff2908, addr=SocketAddr @ 0x00007ee314ff2a70) at adapters.rs:459
    frame #10: 0x0000555d8da366fd grin`_$LT$grin_servers..common..adapters..NetToChainAdapter$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::compact_block_received::h08393e86e10c6756(self=0x00007f631c22ae10, cb=CompactBlock @ 0x00007ee314ff3da0, addr=SocketAddr @ 0x00007ee314ff3f10) at adapters.rs:135
    frame #11: 0x0000555d8e1614bd grin`_$LT$grin_p2p..peers..Peers$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::compact_block_received::hcadb201f6f25adc5(self=0x00007f631c29a3d0, cb=CompactBlock @ 0x00007ee314ff4080, peer_addr=SocketAddr @ 0x00007ee314ff41f0) at peers.rs:525
    frame #12: 0x0000555d8e13ceb7 grin`_$LT$grin_p2p..peer..TrackingAdapter$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::compact_block_received::h8b5832816f41e01b(self=0x00007ee313632010, cb=CompactBlock @ 0x00007ee314ff7ff0, addr=SocketAddr @ 0x00007ee314ff8160) at peer.rs:411
    frame #13: 0x0000555d8e17147c grin`_$LT$grin_p2p..protocol..Protocol$u20$as$u20$grin_p2p..conn..MessageHandler$GT$::consume::haf76e4e6d880f7a2(self=0x00007ee314ffb308, msg=Message @ 0x00007ee314ffaaa0) at protocol.rs:149
    frame #14: 0x0000555d8e0f6910 grin`grin_p2p::conn::poll::_$u7b$$u7b$closure$u7d$$u7d$::h4e7205b8f39a9a73 at conn.rs:221
    frame #15: 0x0000555d8e1904b7 grin`std::sys_common::backtrace::__rust_begin_short_backtrace::ha2469df038c88d8a(f=<unavailable>) at backtrace.rs:136
    frame #16: 0x0000555d8e182872 grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h351813b097efbff7 at mod.rs:409
    frame #17: 0x0000555d8e190325 grin`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::ha61b934b43b96920(self=<unavailable>, _args=<unavailable>) at panic.rs:308
    frame #18: 0x0000555d8e190dba grin`std::panicking::try::do_call::h5684641591dbd454(data="") at panicking.rs:310
    frame #19: 0x0000555d8ebd832a grin`__rust_maybe_catch_panic at lib.rs:105
    frame #20: 0x0000555d8e190baf grin`std::panicking::try::h51e1d65d22999283(f=<unavailable>) at panicking.rs:289
    frame #21: 0x0000555d8e190367 grin`std::panic::catch_unwind::h006e30f8a5c00a9f(f=<unavailable>) at panic.rs:392
    frame #22: 0x0000555d8e182670 grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::hf359b17669af098e at mod.rs:408
    frame #23: 0x0000555d8e182973 grin`_$LT$F$u20$as$u20$alloc..boxed..FnBox$LT$A$GT$$GT$::call_box::h61685e38055643ed(self=0x00007ee313625000, args=<unavailable>) at boxed.rs:640
    frame #24: 0x0000555d8ebb02eb grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 [inlined] _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h75e539106a648d39 at boxed.rs:650
    frame #25: 0x0000555d8ebb02e8 grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 at thread.rs:24
    frame #26: 0x0000555d8ebb3a76 grin`std::sys::unix::thread::Thread::new::thread_start::h7d7a420a78cfa84d at thread.rs:90
    frame #27: 0x00007f631d04a075 libpthread.so.0`start_thread + 229
    frame #28: 0x00007f631cb6753f libc.so.6`__GI___clone + 63

(lldb) thread backtrace 24
  thread #24, name = 'peer'
    frame #0: 0x00007f631d04c57e libpthread.so.0`__pthread_mutex_lock_full + 1086
    frame #1: 0x0000555d8e703bbb grin`mdb_txn_renew0(txn=0x00007f631b74f000) at mdb.c:2692
    frame #2: 0x0000555d8e704d4c grin`mdb_txn_begin(env=0x00007f631c215300, parent=0x0000000000000000, flags=0, ret=0x00007ede10df5a30) at mdb.c:2850
    frame #3: 0x0000555d8e2b24b6 grin`lmdb_zero::tx::ConstTransaction::new::h159de4c669252980(env=Arc<lmdb_zero::env::Environment> @ 0x00007ede10df5a08, parent=Option<&mut lmdb_zero::tx::ConstTransaction> @ 0x00007ede10df5a10, flags=0) at tx.rs:514
    frame #4: 0x0000555d8e2b26ec grin`lmdb_zero::tx::WriteTransaction::new::h9581ef9a44edf59e(env=Arc<lmdb_zero::env::Environment> @ 0x00007ede10df5ae8) at tx.rs:870
    frame #5: 0x0000555d8e2b719b grin`grin_store::lmdb::Store::batch::h02fe585ae6896745(self=0x00007f631c222550) at lmdb.rs:156
    frame #6: 0x0000555d8e1dcaeb grin`grin_chain::store::ChainStore::batch::h29ee00c7c403e5ac(self=0x00007f631c222550) at store.rs:155
    frame #7: 0x0000555d8e21134b grin`grin_chain::chain::Chain::sync_block_headers::h5b030441dadeda39(self=0x00007f631c21b090, headers=0x00007ede10df7130, opts=(bits = 2)) at chain.rs:352
    frame #8: 0x0000555d8da391a9 grin`_$LT$grin_servers..common..adapters..NetToChainAdapter$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::headers_received::h419970ddb8f8a4b9(self=0x00007f631c22ae10, bhs=Vec<grin_core::core::block::BlockHeader> @ 0x00007ede10df7130, addr=SocketAddr @ 0x00007ede10df7150) at adapters.rs:274
    frame #9: 0x0000555d8e161947 grin`_$LT$grin_p2p..peers..Peers$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::headers_received::h0e84f8727db2a9e6(self=0x00007f631c29a3d0, headers=Vec<grin_core::core::block::BlockHeader> @ 0x00007ede10df71d0, peer_addr=SocketAddr @ 0x00007ede10df71f0) at peers.rs:553
    frame #10: 0x0000555d8e13d104 grin`_$LT$grin_p2p..peer..TrackingAdapter$u20$as$u20$grin_p2p..types..ChainAdapter$GT$::headers_received::ha84a5f83e368be22(self=0x00007ede11c34010, bh=Vec<grin_core::core::block::BlockHeader> @ 0x00007ede10dfc050, addr=SocketAddr @ 0x00007ede10dfc070) at peer.rs:420
    frame #11: 0x0000555d8e172141 grin`_$LT$grin_p2p..protocol..Protocol$u20$as$u20$grin_p2p..conn..MessageHandler$GT$::consume::haf76e4e6d880f7a2(self=0x00007ede10dfe308, msg=Message @ 0x00007ede10dfdaa0) at protocol.rs:192
    frame #12: 0x0000555d8e0f6910 grin`grin_p2p::conn::poll::_$u7b$$u7b$closure$u7d$$u7d$::h4e7205b8f39a9a73 at conn.rs:221
    frame #13: 0x0000555d8e1904b7 grin`std::sys_common::backtrace::__rust_begin_short_backtrace::ha2469df038c88d8a(f=<unavailable>) at backtrace.rs:136
    frame #14: 0x0000555d8e182872 grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h351813b097efbff7 at mod.rs:409
    frame #15: 0x0000555d8e190325 grin`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::ha61b934b43b96920(self=<unavailable>, _args=<unavailable>) at panic.rs:308
    frame #16: 0x0000555d8e190dba grin`std::panicking::try::do_call::h5684641591dbd454(data="") at panicking.rs:310
    frame #17: 0x0000555d8ebd832a grin`__rust_maybe_catch_panic at lib.rs:105
    frame #18: 0x0000555d8e190baf grin`std::panicking::try::h51e1d65d22999283(f=<unavailable>) at panicking.rs:289
    frame #19: 0x0000555d8e190367 grin`std::panic::catch_unwind::h006e30f8a5c00a9f(f=<unavailable>) at panic.rs:392
    frame #20: 0x0000555d8e182670 grin`std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::hf359b17669af098e at mod.rs:408
    frame #21: 0x0000555d8e182973 grin`_$LT$F$u20$as$u20$alloc..boxed..FnBox$LT$A$GT$$GT$::call_box::h61685e38055643ed(self=0x00007ede11c27000, args=<unavailable>) at boxed.rs:640
    frame #22: 0x0000555d8ebb02eb grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 [inlined] _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h75e539106a648d39 at boxed.rs:650
    frame #23: 0x0000555d8ebb02e8 grin`std::sys_common::thread::start_thread::h88a639c99862a9f5 at thread.rs:24
    frame #24: 0x0000555d8ebb3a76 grin`std::sys::unix::thread::Thread::new::thread_start::h7d7a420a78cfa84d at thread.rs:90
    frame #25: 0x00007f631d04a075 libpthread.so.0`start_thread + 229
    frame #26: 0x00007f631cb6753f libc.so.6`__GI___clone + 63
(lldb)