)]}'
{
  "commit": "ca2a5e630dc1f569266fb663bf0b65e4eb433fb2",
  "tree": "c4eb62ee6f8c257a42bad5b6fff392251fab80c3",
  "parents": [
    "785abf0dcff0e229aae901485e930f36a8d2ebb2"
  ],
  "author": {
    "name": "Fiona Ebner",
    "email": "f.ebner@proxmox.com",
    "time": "Thu Jul 06 15:14:18 2023 +0200"
  },
  "committer": {
    "name": "Paolo Bonzini",
    "email": "pbonzini@redhat.com",
    "time": "Fri Jul 07 12:49:22 2023 +0200"
  },
  "message": "qemu_cleanup: begin drained section after vm_shutdown()\n\nin order to avoid requests being stuck in a BlockBackend\u0027s request\nqueue during cleanup. Having such requests can lead to a deadlock [0]\nwith a virtio-scsi-pci device using iothread that\u0027s busy with IO when\ninitiating a shutdown with QMP \u0027quit\u0027.\n\nThere is a race where such a queued request can continue sometime\n(maybe after bdrv_child_free()?) during bdrv_root_unref_child() [1].\nThe completion will hold the AioContext lock and wait for the BQL\nduring SCSI completion, but the main thread will hold the BQL and\nwait for the AioContext as part of bdrv_root_unref_child(), leading to\nthe deadlock [0].\n\n[0]:\n\n\u003e Thread 3 (Thread 0x7f3bbd87b700 (LWP 135952) \"qemu-system-x86\"):\n\u003e #0  __lll_lock_wait (futex\u003dfutex@entry\u003d0x564183365f00 \u003cqemu_global_mutex\u003e, private\u003d0) at lowlevellock.c:52\n\u003e #1  0x00007f3bc1c0d843 in __GI___pthread_mutex_lock (mutex\u003d0x564183365f00 \u003cqemu_global_mutex\u003e) at ../nptl/pthread_mutex_lock.c:80\n\u003e #2  0x0000564182939f2e in qemu_mutex_lock_impl (mutex\u003d0x564183365f00 \u003cqemu_global_mutex\u003e, file\u003d0x564182b7f774 \"../softmmu/physmem.c\", line\u003d2593) at ../util/qemu-thread-posix.c:94\n\u003e #3  0x000056418247cc2a in qemu_mutex_lock_iothread_impl (file\u003d0x564182b7f774 \"../softmmu/physmem.c\", line\u003d2593) at ../softmmu/cpus.c:504\n\u003e #4  0x00005641826d5325 in prepare_mmio_access (mr\u003d0x5641856148a0) at ../softmmu/physmem.c:2593\n\u003e #5  0x00005641826d6fe7 in address_space_stl_internal (as\u003d0x56418679b310, addr\u003d4276113408, val\u003d16418, attrs\u003d..., result\u003d0x0, endian\u003dDEVICE_LITTLE_ENDIAN) at /home/febner/repos/qemu/memory_ldst.c.inc:318\n\u003e #6  0x00005641826d7154 in address_space_stl_le (as\u003d0x56418679b310, addr\u003d4276113408, val\u003d16418, attrs\u003d..., result\u003d0x0) at /home/febner/repos/qemu/memory_ldst.c.inc:357\n\u003e #7  0x0000564182374b07 in pci_msi_trigger (dev\u003d0x56418679b0d0, msg\u003d...) at ../hw/pci/pci.c:359\n\u003e #8  0x000056418237118b in msi_send_message (dev\u003d0x56418679b0d0, msg\u003d...) at ../hw/pci/msi.c:379\n\u003e #9  0x0000564182372c10 in msix_notify (dev\u003d0x56418679b0d0, vector\u003d8) at ../hw/pci/msix.c:542\n\u003e #10 0x000056418243719c in virtio_pci_notify (d\u003d0x56418679b0d0, vector\u003d8) at ../hw/virtio/virtio-pci.c:77\n\u003e #11 0x00005641826933b0 in virtio_notify_vector (vdev\u003d0x5641867a34a0, vector\u003d8) at ../hw/virtio/virtio.c:1985\n\u003e #12 0x00005641826948d6 in virtio_irq (vq\u003d0x5641867ac078) at ../hw/virtio/virtio.c:2461\n\u003e #13 0x0000564182694978 in virtio_notify (vdev\u003d0x5641867a34a0, vq\u003d0x5641867ac078) at ../hw/virtio/virtio.c:2473\n\u003e #14 0x0000564182665b83 in virtio_scsi_complete_req (req\u003d0x7f3bb000e5d0) at ../hw/scsi/virtio-scsi.c:115\n\u003e #15 0x00005641826670ce in virtio_scsi_complete_cmd_req (req\u003d0x7f3bb000e5d0) at ../hw/scsi/virtio-scsi.c:641\n\u003e #16 0x000056418266736b in virtio_scsi_command_complete (r\u003d0x7f3bb0010560, resid\u003d0) at ../hw/scsi/virtio-scsi.c:712\n\u003e #17 0x000056418239aac6 in scsi_req_complete (req\u003d0x7f3bb0010560, status\u003d2) at ../hw/scsi/scsi-bus.c:1526\n\u003e #18 0x000056418239e090 in scsi_handle_rw_error (r\u003d0x7f3bb0010560, ret\u003d-123, acct_failed\u003dfalse) at ../hw/scsi/scsi-disk.c:242\n\u003e #19 0x000056418239e13f in scsi_disk_req_check_error (r\u003d0x7f3bb0010560, ret\u003d-123, acct_failed\u003dfalse) at ../hw/scsi/scsi-disk.c:265\n\u003e #20 0x000056418239e482 in scsi_dma_complete_noio (r\u003d0x7f3bb0010560, ret\u003d-123) at ../hw/scsi/scsi-disk.c:340\n\u003e #21 0x000056418239e5d9 in scsi_dma_complete (opaque\u003d0x7f3bb0010560, ret\u003d-123) at ../hw/scsi/scsi-disk.c:371\n\u003e #22 0x00005641824809ad in dma_complete (dbs\u003d0x7f3bb000d9d0, ret\u003d-123) at ../softmmu/dma-helpers.c:107\n\u003e #23 0x0000564182480a72 in dma_blk_cb (opaque\u003d0x7f3bb000d9d0, ret\u003d-123) at ../softmmu/dma-helpers.c:127\n\u003e #24 0x00005641827bf78a in blk_aio_complete (acb\u003d0x7f3bb00021a0) at ../block/block-backend.c:1563\n\u003e #25 0x00005641827bfa5e in blk_aio_write_entry (opaque\u003d0x7f3bb00021a0) at ../block/block-backend.c:1630\n\u003e #26 0x000056418295638a in coroutine_trampoline (i0\u003d-1342102448, i1\u003d32571) at ../util/coroutine-ucontext.c:177\n\u003e #27 0x00007f3bc0caed40 in ?? () from /lib/x86_64-linux-gnu/libc.so.6\n\u003e #28 0x00007f3bbd8757f0 in ?? ()\n\u003e #29 0x0000000000000000 in ?? ()\n\u003e\n\u003e Thread 1 (Thread 0x7f3bbe3e9280 (LWP 135944) \"qemu-system-x86\"):\n\u003e #0  __lll_lock_wait (futex\u003dfutex@entry\u003d0x5641856f2a00, private\u003d0) at lowlevellock.c:52\n\u003e #1  0x00007f3bc1c0d8d1 in __GI___pthread_mutex_lock (mutex\u003d0x5641856f2a00) at ../nptl/pthread_mutex_lock.c:115\n\u003e #2  0x0000564182939f2e in qemu_mutex_lock_impl (mutex\u003d0x5641856f2a00, file\u003d0x564182c0e319 \"../util/async.c\", line\u003d728) at ../util/qemu-thread-posix.c:94\n\u003e #3  0x000056418293a140 in qemu_rec_mutex_lock_impl (mutex\u003d0x5641856f2a00, file\u003d0x564182c0e319 \"../util/async.c\", line\u003d728) at ../util/qemu-thread-posix.c:149\n\u003e #4  0x00005641829532d5 in aio_context_acquire (ctx\u003d0x5641856f29a0) at ../util/async.c:728\n\u003e #5  0x000056418279d5df in bdrv_set_aio_context_commit (opaque\u003d0x5641856e6e50) at ../block.c:7493\n\u003e #6  0x000056418294e288 in tran_commit (tran\u003d0x56418630bfe0) at ../util/transactions.c:87\n\u003e #7  0x000056418279d880 in bdrv_try_change_aio_context (bs\u003d0x5641856f7130, ctx\u003d0x56418548f810, ignore_child\u003d0x0, errp\u003d0x0) at ../block.c:7626\n\u003e #8  0x0000564182793f39 in bdrv_root_unref_child (child\u003d0x5641856f47d0) at ../block.c:3242\n\u003e #9  0x00005641827be137 in blk_remove_bs (blk\u003d0x564185709880) at ../block/block-backend.c:914\n\u003e #10 0x00005641827bd689 in blk_remove_all_bs () at ../block/block-backend.c:583\n\u003e #11 0x0000564182798699 in bdrv_close_all () at ../block.c:5117\n\u003e #12 0x000056418248a5b2 in qemu_cleanup () at ../softmmu/runstate.c:821\n\u003e #13 0x0000564182738603 in qemu_default_main () at ../softmmu/main.c:38\n\u003e #14 0x0000564182738631 in main (argc\u003d30, argv\u003d0x7ffd675a8a48) at ../softmmu/main.c:48\n\u003e\n\u003e (gdb) p *((QemuMutex*)0x5641856f2a00)\n\u003e $1 \u003d {lock \u003d {__data \u003d {__lock \u003d 2, __count \u003d 2, __owner \u003d 135952, ...\n\u003e (gdb) p *((QemuMutex*)0x564183365f00)\n\u003e $2 \u003d {lock \u003d {__data \u003d {__lock \u003d 2, __count \u003d 0, __owner \u003d 135944, ...\n\n[1]:\n\n\u003e Thread 1 \"qemu-system-x86\" hit Breakpoint 5, bdrv_drain_all_end () at ../block/io.c:551\n\u003e #0  bdrv_drain_all_end () at ../block/io.c:551\n\u003e #1  0x00005569810f0376 in bdrv_graph_wrlock (bs\u003d0x0) at ../block/graph-lock.c:156\n\u003e #2  0x00005569810bd3e0 in bdrv_replace_child_noperm (child\u003d0x556982e2d7d0, new_bs\u003d0x0) at ../block.c:2897\n\u003e #3  0x00005569810bdef2 in bdrv_root_unref_child (child\u003d0x556982e2d7d0) at ../block.c:3227\n\u003e #4  0x00005569810e8137 in blk_remove_bs (blk\u003d0x556982e42880) at ../block/block-backend.c:914\n\u003e #5  0x00005569810e7689 in blk_remove_all_bs () at ../block/block-backend.c:583\n\u003e #6  0x00005569810c2699 in bdrv_close_all () at ../block.c:5117\n\u003e #7  0x0000556980db45b2 in qemu_cleanup () at ../softmmu/runstate.c:821\n\u003e #8  0x0000556981062603 in qemu_default_main () at ../softmmu/main.c:38\n\u003e #9  0x0000556981062631 in main (argc\u003d30, argv\u003d0x7ffd7a82a418) at ../softmmu/main.c:48\n\u003e [Switching to Thread 0x7fe76dab2700 (LWP 103649)]\n\u003e\n\u003e Thread 3 \"qemu-system-x86\" hit Breakpoint 4, blk_inc_in_flight (blk\u003d0x556982e42880) at ../block/block-backend.c:1505\n\u003e #0  blk_inc_in_flight (blk\u003d0x556982e42880) at ../block/block-backend.c:1505\n\u003e #1  0x00005569810e8f36 in blk_wait_while_drained (blk\u003d0x556982e42880) at ../block/block-backend.c:1312\n\u003e #2  0x00005569810e9231 in blk_co_do_pwritev_part (blk\u003d0x556982e42880, offset\u003d3422961664, bytes\u003d4096, qiov\u003d0x556983028060, qiov_offset\u003d0, flags\u003d0) at ../block/block-backend.c:1402\n\u003e #3  0x00005569810e9a4b in blk_aio_write_entry (opaque\u003d0x556982e2cfa0) at ../block/block-backend.c:1628\n\u003e #4  0x000055698128038a in coroutine_trampoline (i0\u003d-2090057872, i1\u003d21865) at ../util/coroutine-ucontext.c:177\n\u003e #5  0x00007fe770f50d40 in ?? () from /lib/x86_64-linux-gnu/libc.so.6\n\u003e #6  0x00007ffd7a829570 in ?? ()\n\u003e #7  0x0000000000000000 in ?? ()\n\nSigned-off-by: Fiona Ebner \u003cf.ebner@proxmox.com\u003e\nMessage-ID: \u003c20230706131418.423713-1-f.ebner@proxmox.com\u003e\nSigned-off-by: Paolo Bonzini \u003cpbonzini@redhat.com\u003e\n",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "a9fbcf48622f3bb5d287377c020e13c475b04ded",
      "old_mode": 33188,
      "old_path": "softmmu/runstate.c",
      "new_id": "f3bd8628181303792629fa4079f09abf63fd9787",
      "new_mode": 33188,
      "new_path": "softmmu/runstate.c"
    }
  ]
}