There are two follow on changes to this code (in the not yet released qemu 6.0):
 050de36b13 coroutine-lock: Reimplement CoRwlock to fix downgrade bug
 2f6ef0393b coroutine-lock: Store the coroutine in the CoWaitRecord only once

They change how things are done, but are no known fixes to the current issue.

We might gather more data and report it upstream - it could ring a bell for
someone there.

Attaching gdb to the live qemu in into further issues
# Cannot find user-level thread for LWP 29341: generic error
Which on qemu led to
# [  172.294630] watchdog: BUG: soft lockup - CPU#0 stuck for 78s! [systemd-udevd:173]

I'm not sorting this out now, so post mortem debugging it will be :-/

I've taken a crash dump of the most recent 1:5.2+dfsg-9ubuntu2 which
has debug symbols in Ubuntu and even later one can fetch from
https://launchpad.net/ubuntu/+source/qemu/1:5.2+dfsg-9ubuntu2

(gdb) info threads
  Id   Target Id                         Frame 
* 1    Thread 0xffffa98f9010 (LWP 29397) __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:49
  2    Thread 0xffffa904f8b0 (LWP 29398) syscall () at ../sysdeps/unix/sysv/linux/aarch64/syscall.S:38
  3    Thread 0xffffa3ffe8b0 (LWP 29399) 0x0000ffffab022d14 in __GI___sigtimedwait (set=set@entry=0xaaaac2fed320, info=info@entry=0xffffa3ffdd88, timeout=timeout@entry=0x0)
    at ../sysdeps/unix/sysv/linux/sigtimedwait.c:54
  4    Thread 0xffff237ee8b0 (LWP 29407) __futex_abstimed_wait_common64 (cancel=true, private=-1022925096, abstime=0xffff237ede48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766d8)
    at ../sysdeps/nptl/futex-internal.c:74
  5    Thread 0xffff22fde8b0 (LWP 29408) __futex_abstimed_wait_common64 (cancel=true, private=-1022925096, abstime=0xffff22fdde48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766d8)
    at ../sysdeps/nptl/futex-internal.c:74
  6    Thread 0xffff2bee18b0 (LWP 29405) __futex_abstimed_wait_common64 (cancel=true, private=-1022925092, abstime=0xffff2bee0e48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766dc)
    at ../sysdeps/nptl/futex-internal.c:74
  7    Thread 0xffffa27ce8b0 (LWP 29402) futex_wait (private=0, expected=2, futex_word=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at ../sysdeps/nptl/futex-internal.h:146
  8    Thread 0xffffa2fde8b0 (LWP 29401) futex_wait (private=0, expected=2, futex_word=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at ../sysdeps/nptl/futex-internal.h:146
  9    Thread 0xffff23ffe8b0 (LWP 29406) 0x0000ffffab0b9024 in __GI_pwritev64 (fd=<optimized out>, vector=0xaaaac3559fd0, count=2, offset=668794880)
    at ../sysdeps/unix/sysv/linux/pwritev64.c:26
  10   Thread 0xffffa37ee8b0 (LWP 29404) 0x0000ffffab0b9d3c in fdatasync (fd=<optimized out>) at ../sysdeps/unix/sysv/linux/fdatasync.c:28

(gdb) thread apply all bt

Thread 10 (Thread 0xffffa37ee8b0 (LWP 29404)):
#0  0x0000ffffab0b9d3c in fdatasync (fd=<optimized out>) at ../sysdeps/unix/sysv/linux/fdatasync.c:28
#1  0x0000aaaab8b8d3a8 in qemu_fdatasync (fd=<optimized out>) at ../../util/cutils.c:161
#2  handle_aiocb_flush (opaque=<optimized out>) at ../../block/file-posix.c:1350
#3  0x0000aaaab8c57314 in worker_thread (opaque=opaque@entry=0xaaaac3076600) at ../../util/thread-pool.c:104
#4  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#5  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#6  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 9 (Thread 0xffff23ffe8b0 (LWP 29406)):
#0  0x0000ffffab0b9024 in __GI_pwritev64 (fd=<optimized out>, vector=0xaaaac3559fd0, count=2, offset=668794880) at ../sysdeps/unix/sysv/linux/pwritev64.c:26
#1  0x0000aaaab8b91418 in qemu_pwritev (offset=<optimized out>, nr_iov=<optimized out>, iov=<optimized out>, fd=<optimized out>) at ../../block/file-posix.c:1386
#2  handle_aiocb_rw_vector (aiocb=<optimized out>) at ../../block/file-posix.c:1413
#3  handle_aiocb_rw (opaque=0xffffa0e2bba8) at ../../block/file-posix.c:1498
#4  0x0000aaaab8c57314 in worker_thread (opaque=opaque@entry=0xaaaac3076600) at ../../util/thread-pool.c:104
#5  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#6  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#7  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 8 (Thread 0xffffa2fde8b0 (LWP 29401)):
#0  futex_wait (private=0, expected=2, futex_word=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at ../sysdeps/nptl/futex-internal.h:146
#1  __lll_lock_wait (futex=futex@entry=0xaaaab912d640 <qemu_global_mutex.lto_priv>, private=0) at lowlevellock.c:52
#2  0x0000ffffab170fd8 in __GI___pthread_mutex_lock (mutex=mutex@entry=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at pthread_mutex_lock.c:80
#3  0x0000aaaab8c72dac in qemu_mutex_lock_impl (mutex=0xaaaab912d640 <qemu_global_mutex.lto_priv>, file=0xaaaab8d77980 "../../accel/tcg/cpu-exec.c", line=509) at ../../util/qemu-thread-posix.c:79
#4  0x0000aaaab8a26010 in qemu_mutex_lock_iothread_impl (file=<optimized out>, line=<optimized out>) at ../../softmmu/cpus.c:485
#5  0x0000aaaab8a70794 in cpu_handle_exception (ret=<synthetic pointer>, cpu=0xaaaac31bda80) at ../../accel/tcg/cpu-exec.c:509
#6  cpu_exec (cpu=0xaaaac31bda80) at ../../accel/tcg/cpu-exec.c:751
#7  0x0000aaaab8a3dd40 in tcg_cpu_exec (cpu=0xaaaac31bda80) at ../../accel/tcg/tcg-cpus.c:243
#8  tcg_cpu_thread_fn (arg=arg@entry=0xaaaac31bda80) at ../../accel/tcg/tcg-cpus.c:427
#9  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#10 0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#11 0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 7 (Thread 0xffffa27ce8b0 (LWP 29402)):
#0  futex_wait (private=0, expected=2, futex_word=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at ../sysdeps/nptl/futex-internal.h:146
#1  __lll_lock_wait (futex=futex@entry=0xaaaab912d640 <qemu_global_mutex.lto_priv>, private=0) at lowlevellock.c:52
#2  0x0000ffffab170fd8 in __GI___pthread_mutex_lock (mutex=mutex@entry=0xaaaab912d640 <qemu_global_mutex.lto_priv>) at pthread_mutex_lock.c:80
#3  0x0000aaaab8c72dac in qemu_mutex_lock_impl (mutex=0xaaaab912d640 <qemu_global_mutex.lto_priv>, file=0xaaaab8d77980 "../../accel/tcg/cpu-exec.c", line=563) at ../../util/qemu-thread-posix.c:79
#4  0x0000aaaab8a26010 in qemu_mutex_lock_iothread_impl (file=<optimized out>, line=<optimized out>) at ../../softmmu/cpus.c:485
--Type <RET> for more, q to quit, c to continue without paging--
#5  0x0000aaaab8a70e3c in cpu_handle_interrupt (last_tb=<synthetic pointer>, cpu=0xaaaac31c76c0) at ../../accel/tcg/cpu-exec.c:563
#6  cpu_exec (cpu=0xaaaac31c76c0) at ../../accel/tcg/cpu-exec.c:755
#7  0x0000aaaab8a3dd40 in tcg_cpu_exec (cpu=0xaaaac31c76c0) at ../../accel/tcg/tcg-cpus.c:243
#8  tcg_cpu_thread_fn (arg=arg@entry=0xaaaac31c76c0) at ../../accel/tcg/tcg-cpus.c:427
#9  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#10 0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#11 0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 6 (Thread 0xffff2bee18b0 (LWP 29405)):
#0  __futex_abstimed_wait_common64 (cancel=true, private=-1022925092, abstime=0xffff2bee0e48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766dc) at ../sysdeps/nptl/futex-internal.c:74
#1  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0xaaaac30766dc, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0xffff2bee0e48, private=private@entry=0) at ../sysdeps/nptl/futex-internal.c:123
#2  0x0000ffffab1757e4 in __pthread_cond_wait_common (abstime=0xffff2bee0e48, clockid=0, mutex=0xaaaac3076680, cond=0xaaaac30766b0) at pthread_cond_wait.c:504
#3  __pthread_cond_timedwait (cond=cond@entry=0xaaaac30766b0, mutex=mutex@entry=0xaaaac3076680, abstime=abstime@entry=0xffff2bee0e48) at pthread_cond_wait.c:646
#4  0x0000aaaab8c6cc60 in qemu_sem_timedwait (sem=0xaaaac3076680, ms=<optimized out>) at ../../util/qemu-thread-posix.c:282
#5  0x0000aaaab8c572a4 in worker_thread (opaque=opaque@entry=0xaaaac3076600) at ../../util/thread-pool.c:91
#6  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#7  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#8  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 5 (Thread 0xffff22fde8b0 (LWP 29408)):
#0  __futex_abstimed_wait_common64 (cancel=true, private=-1022925096, abstime=0xffff22fdde48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766d8) at ../sysdeps/nptl/futex-internal.c:74
#1  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0xaaaac30766d8, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0xffff22fdde48, private=private@entry=0) at ../sysdeps/nptl/futex-internal.c:123
#2  0x0000ffffab1757e4 in __pthread_cond_wait_common (abstime=0xffff22fdde48, clockid=0, mutex=0xaaaac3076680, cond=0xaaaac30766b0) at pthread_cond_wait.c:504
#3  __pthread_cond_timedwait (cond=cond@entry=0xaaaac30766b0, mutex=mutex@entry=0xaaaac3076680, abstime=abstime@entry=0xffff22fdde48) at pthread_cond_wait.c:646
#4  0x0000aaaab8c6cc60 in qemu_sem_timedwait (sem=0xaaaac3076680, ms=<optimized out>) at ../../util/qemu-thread-posix.c:282
#5  0x0000aaaab8c572a4 in worker_thread (opaque=opaque@entry=0xaaaac3076600) at ../../util/thread-pool.c:91
#6  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#7  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#8  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 4 (Thread 0xffff237ee8b0 (LWP 29407)):
#0  __futex_abstimed_wait_common64 (cancel=true, private=-1022925096, abstime=0xffff237ede48, clockid=-1022925184, expected=0, futex_word=0xaaaac30766d8) at ../sysdeps/nptl/futex-internal.c:74
#1  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0xaaaac30766d8, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0xffff237ede48, private=private@entry=0) at ../sysdeps/nptl/futex-internal.c:123
#2  0x0000ffffab1757e4 in __pthread_cond_wait_common (abstime=0xffff237ede48, clockid=0, mutex=0xaaaac3076680, cond=0xaaaac30766b0) at pthread_cond_wait.c:504
#3  __pthread_cond_timedwait (cond=cond@entry=0xaaaac30766b0, mutex=mutex@entry=0xaaaac3076680, abstime=abstime@entry=0xffff237ede48) at pthread_cond_wait.c:646
#4  0x0000aaaab8c6cc60 in qemu_sem_timedwait (sem=0xaaaac3076680, ms=<optimized out>) at ../../util/qemu-thread-posix.c:282
--Type <RET> for more, q to quit, c to continue without paging--
#5  0x0000aaaab8c572a4 in worker_thread (opaque=opaque@entry=0xaaaac3076600) at ../../util/thread-pool.c:91
#6  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#7  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#8  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 3 (Thread 0xffffa3ffe8b0 (LWP 29399)):
#0  0x0000ffffab022d14 in __GI___sigtimedwait (set=set@entry=0xaaaac2fed320, info=info@entry=0xffffa3ffdd88, timeout=timeout@entry=0x0) at ../sysdeps/unix/sysv/linux/sigtimedwait.c:54
#1  0x0000ffffab17a180 in __sigwait (set=set@entry=0xaaaac2fed320, sig=sig@entry=0xffffa3ffde54) at ../sysdeps/unix/sysv/linux/sigwait.c:28
#2  0x0000aaaab8c6a834 in sigwait_compat (opaque=opaque@entry=0xaaaac2fed320) at ../../util/compatfd.c:37
#3  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#4  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#5  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 2 (Thread 0xffffa904f8b0 (LWP 29398)):
#0  syscall () at ../sysdeps/unix/sysv/linux/aarch64/syscall.S:38
#1  0x0000aaaab8c6cf10 in qemu_futex_wait (val=<optimized out>, f=<optimized out>) at ./include/qemu/futex.h:29
#2  qemu_event_wait (ev=0xaaaab9149580 <rcu_call_ready_event.lto_priv>) at ../../util/qemu-thread-posix.c:460
#3  0x0000aaaab8c60734 in call_rcu_thread (opaque=opaque@entry=0x0) at ../../util/rcu.c:258
#4  0x0000aaaab8c69f88 in qemu_thread_start (args=<optimized out>) at ../../util/qemu-thread-posix.c:521
#5  0x0000ffffab16ee84 in start_thread (arg=0x0) at pthread_create.c:473
#6  0x0000ffffab0c0d1c in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:77

Thread 1 (Thread 0xffffa98f9010 (LWP 29397)):
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:49
#1  0x0000ffffab00e6cc in __GI_abort () at abort.c:79
#2  0x0000ffffab01b318 in __assert_fail_base (fmt=0xffffab119208 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0xaaaab8d9c628 "qemu_coroutine_self() == pool->main_co", file=file@entry=0xaaaab8d9c5b8 "../../block/aio_task.c", line=line@entry=64, function=function@entry=0xaaaab8d9e350 <__PRETTY_FUNCTION__.1.lto_priv.62> "aio_task_pool_wait_one") at assert.c:92
#3  0x0000ffffab01b37c in __GI___assert_fail (assertion=assertion@entry=0xaaaab8d9c628 "qemu_coroutine_self() == pool->main_co", file=file@entry=0xaaaab8d9c5b8 "../../block/aio_task.c", line=line@entry=64, function=function@entry=0xaaaab8d9e350 <__PRETTY_FUNCTION__.1.lto_priv.62> "aio_task_pool_wait_one") at assert.c:101
#4  0x0000aaaab8b6bf58 in aio_task_pool_wait_one (pool=0xaaaac34987b0) at ../../block/aio_task.c:64
#5  0x0000aaaab8ba586c in aio_task_pool_wait_all (pool=<optimized out>) at ../../block/aio_task.c:85
#6  qcow2_co_pwritev_part (bs=0xaaaac318ce50, offset=<optimized out>, bytes=<optimized out>, qiov=0xffff98397298, qiov_offset=<optimized out>, flags=<optimized out>) at ../../block/qcow2.c:2638
#7  0x0000aaaab8b32b60 in bdrv_driver_pwritev (bs=bs@entry=0xaaaac318ce50, offset=offset@entry=614723584, bytes=bytes@entry=1310720, qiov=qiov@entry=0xffff98397298, qiov_offset=qiov_offset@entry=0, flags=flags@entry=0) at ../../block/io.c:1114
#8  0x0000aaaab8b34c54 in bdrv_aligned_pwritev (child=0xaaaac3176100, req=0xffffa0b28e48, offset=614723584, bytes=1310720, align=<optimized out>, qiov=0xffff98397298, qiov_offset=0, flags=0) at ../../block/io.c:1936
#9  0x0000aaaab8b35ad0 in bdrv_co_pwritev_part (child=0xaaaac3176100, offset=<optimized out>, bytes=<optimized out>, qiov=<optimized out>, qiov_offset=<optimized out>, flags=0) at ../../block/io.c:2104
#10 0x0000aaaab8b84220 in blk_do_pwritev_part (blk=0xaaaac318caf0, offset=614723584, bytes=1310720, qiov=0xffff98397298, qiov_offset=qiov_offset@entry=0, flags=<optimized out>) at ../../block/block-backend.c:1260
--Type <RET> for more, q to quit, c to continue without paging--
#11 0x0000aaaab8b84530 in blk_aio_write_entry (opaque=0xffff987aa7a0) at ../../block/block-backend.c:1476
#12 0x0000aaaab8c3b980 in coroutine_trampoline (i0=<optimized out>, i1=<optimized out>) at ../../util/coroutine-ucontext.c:173
#13 0x0000ffffab031080 in ?? () at ../sysdeps/unix/sysv/linux/aarch64/setcontext.S:123 from /lib/aarch64-linux-gnu/libc.so.6
Backtrace stopped: not enough registers or memory available to unwind further

I'll keep the file around but with 1.8GB I need a better place if I want to
share the dump file itself :-/