LP #1505564 - IS - Kernel Lockups - 3.13.0-24-generic kernel dump analysis #### #### sys #### SYSTEM MAP: /boot/System.map-3.13.0-24-generic DEBUG KERNEL: /usr/lib/debug/boot/vmlinux-3.13.0-24-generic (3.13.0-24-generic) DUMPFILE: ./dump.201510281618 [PARTIAL DUMP] CPUS: 32 DATE: Wed Oct 28 16:18:20 2015 UPTIME: 07:27:20 LOAD AVERAGE: 56.66, 33.23, 30.00 TASKS: 892 NODENAME: druk RELEASE: 3.13.0-24-generic VERSION: #47-Ubuntu SMP Fri May 2 23:30:00 UTC 2014 MACHINE: x86_64 (2600 Mhz) MEMORY: 128 GB PANIC: "Kernel panic - not syncing: softlockup: hung tasks" PID: 123 COMMAND: "migration/10" TASK: ffff88081404c7d0 [THREAD_INFO: ffff880814054000] CPU: 10 STATE: TASK_RUNNING (PANIC) #### #### panic’ed task #### crash> bt PID: 123 TASK: ffff88081404c7d0 CPU: 10 COMMAND: "migration/10" #0 [ffff88081fd43d18] machine_kexec at ffffffff8104a732 #1 [ffff88081fd43d68] crash_kexec at ffffffff810e6ab3 #2 [ffff88081fd43e30] panic at ffffffff8170eccc #3 [ffff88081fd43ea8] watchdog_timer_fn at ffffffff8110d015 #4 [ffff88081fd43ed8] __run_hrtimer at ffffffff8108e537 #5 [ffff88081fd43f18] hrtimer_interrupt at ffffffff8108ed3f #6 [ffff88081fd43f80] local_apic_timer_interrupt at ffffffff81043087 #7 [ffff88081fd43f98] smp_apic_timer_interrupt at ffffffff8172887f #8 [ffff88081fd43fb0] apic_timer_interrupt at ffffffff8172721d --- --- #9 [ffff880814055ce8] apic_timer_interrupt at ffffffff8172721d [exception RIP: multi_cpu_stop+107] RIP: ffffffff810f571b RSP: ffff880814055d98 RFLAGS: 00000293 RAX: ffffffff8180af40 RBX: 0000000000000082 RCX: 0000000000000740 RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff8801a7809798 RBP: ffff880814055db8 R8: 0000000000000282 R9: 0000000000000001 R10: 000000000000b8ee R11: ffff880ab3f9cd80 R12: 0000000000014440 R13: ffff880814140a00 R14: ffff880814140a08 R15: 000000020000000a ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018 #10 [ffff880814055dc0] cpu_stopper_thread at ffffffff810f592e #11 [ffff880814055e80] smpboot_thread_fn at ffffffff8109263f #12 [ffff880814055ed0] kthread at ffffffff8108b312 #13 [ffff880814055f50] ret_from_fork at ffffffff817263fc #### #### active tasks #### ### swapper tasks (idling) > 0 0 2 ffff8808147297f0 RU 0.0 0 0 [swapper/2] > 0 0 7 ffff8808147397f0 RU 0.0 0 0 [swapper/7] > 0 0 14 ffff8808147547d0 RU 0.0 0 0 [swapper/14] > 0 0 22 ffff8808147717f0 RU 0.0 0 0 [swapper/22] > 0 0 25 ffff880814775fc0 RU 0.0 0 0 [swapper/25] > 0 0 26 ffff880814780000 RU 0.0 0 0 [swapper/26] > 0 0 28 ffff880814782fe0 RU 0.0 0 0 [swapper/28] > 0 0 29 ffff8808147847d0 RU 0.0 0 0 [swapper/29] > 0 0 30 ffff880814785fc0 RU 0.0 0 0 [swapper/30] **** All tasks are on the same point: IO port based C-state crash> bt ffff8808147297f0 PID: 0 TASK: ffff8808147297f0 CPU: 2 COMMAND: "swapper/2" #0 [ffff88081fc46e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88081fc46e68] nmi_handle at ffffffff8171f188 #2 [ffff88081fc46ec8] do_nmi at ffffffff8171f350 #3 [ffff88081fc46ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: acpi_idle_do_entry+33] RIP: ffffffff81412502 RSP: ffff880814731e28 RFLAGS: 00000093 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000093 RDX: ffff880814731e28 RSI: 0000000000000018 RDI: 0000000000000001 RBP: ffffffff81412502 R8: ffffffff81412502 R9: 0000000000000018 R10: ffff880814731e28 R11: 0000000000000093 R12: ffffffffffffffff R13: ffff880814261ca8 R14: ffff88081fc40000 R15: 0000000000000ff9 ORIG_RAX: 0000000000000ff9 CS: 0010 SS: 0018 --- --- #4 [ffff880814731e28] acpi_idle_do_entry at ffffffff81412502 700 } else { 701 /* IO port based C-state */ 702 inb(cx->address); 703 /* Dummy wait op - must do something useless after P_LVL2 read 704 because chipsets cannot guarantee that STPCLK# signal 705 gets asserted in time to freeze execution properly. */ 706 inl(acpi_gbl_FADT.xpm_timer_block.address); 0xffffffff81412502 <+33>: mov 0xbac76b(%rip),%rdx # 0xffffffff81fbec74 #5 [ffff880814731e30] acpi_idle_enter_simple at ffffffff81412587 #6 [ffff880814731e58] cpuidle_enter_state at ffffffff815c95d0 #7 [ffff880814731e90] cpuidle_idle_call at ffffffff815c9709 #8 [ffff880814731ed0] arch_cpu_idle at ffffffff8101ceae #9 [ffff880814731ee0] cpu_startup_entry at ffffffff810beb85 #10 [ffff880814731f30] start_secondary at ffffffff81040fc8 #### migration tasks (state machine loop) > 93 2 4 ffff8808147b47d0 RU 0.0 0 0 [migration/4] > 118 2 9 ffff881814a2c7d0 RU 0.0 0 0 [migration/9] > 123 2 10 ffff88081404c7d0 RU 0.0 0 0 [migration/10] > 128 2 11 ffff881814a4c7d0 RU 0.0 0 0 [migration/11] > 138 2 13 ffff881814a647d0 RU 0.0 0 0 [migration/13] > 165 2 18 ffff8810149ec7d0 RU 0.0 0 0 [migration/18] > 195 2 24 ffff881014a647d0 RU 0.0 0 0 [migration/24] crash> bt ffff8808147b47d0 PID: 93 TASK: ffff8808147b47d0 CPU: 4 COMMAND: "migration/4" #0 [ffff88081fc86e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88081fc86e68] nmi_handle at ffffffff8171f188 #2 [ffff88081fc86ec8] do_nmi at ffffffff8171f350 #3 [ffff88081fc86ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: multi_cpu_stop+107] RIP: ffffffff810f571b RSP: ffff8808147fbd98 RFLAGS: 00000293 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000293 RDX: ffff8808147fbd98 RSI: 0000000000000000 RDI: 0000000000000001 RBP: ffffffff810f571b R8: ffffffff810f571b R9: 0000000000000000 R10: ffff8808147fbd98 R11: 0000000000000293 R12: ffffffffffffffff R13: ffff880c24033b48 R14: 0000000000000000 R15: 0000000000000001 ORIG_RAX: 0000000000000001 CS: 0010 SS: 0000 --- --- #4 [ffff8808147fbd98] multi_cpu_stop at ffffffff810f571b #5 [ffff8808147fbdc0] cpu_stopper_thread at ffffffff810f592e #6 [ffff8808147fbe80] smpboot_thread_fn at ffffffff8109263f #7 [ffff8808147fbed0] kthread at ffffffff8108b312 #8 [ffff8808147fbf50] ret_from_fork at ffffffff817263fc 188 /* Simple state machine */ 189 do { 190 /* Chill out and ensure we re-read multi_stop_state. */ 191 cpu_relax(); 192 if (msdata->state != curstate) { 0xffffffff810f571b <+107>: mov 0x20(%r12),%ebx 0xffffffff810f5720 <+112>: cmp %edx,%ebx 0xffffffff810f5722 <+114>: je 0xffffffff810f5714 ---- > 133 2 12 ffff88081407c7d0 RU 0.0 0 0 [migration/12] crash> bt ffff88081407c7d0 PID: 133 TASK: ffff88081407c7d0 CPU: 12 COMMAND: "migration/12" #0 [ffff88081fd86e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88081fd86e68] nmi_handle at ffffffff8171f188 #2 [ffff88081fd86ec8] do_nmi at ffffffff8171f350 #3 [ffff88081fd86ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: multi_cpu_stop+100] RIP: ffffffff810f5714 RSP: ffff88081408dd98 RFLAGS: 00000246 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000246 RDX: ffff88081408dd98 RSI: 0000000000000000 RDI: 0000000000000001 RBP: ffffffff810f5714 R8: ffffffff810f5714 R9: 0000000000000000 R10: ffff88081408dd98 R11: 0000000000000246 R12: ffffffffffffffff R13: ffff88184368d798 R14: 0000000000000000 R15: 0000000000000001 ORIG_RAX: 0000000000000001 CS: 0010 SS: 0000 --- --- #4 [ffff88081408dd98] multi_cpu_stop at ffffffff810f5714 #5 [ffff88081408ddc0] cpu_stopper_thread at ffffffff810f592e #6 [ffff88081408de80] smpboot_thread_fn at ffffffff8109263f #7 [ffff88081408ded0] kthread at ffffffff8108b312 #8 [ffff88081408df50] ret_from_fork at ffffffff817263fc 202 break; 203 default: 204 break; 205 } 206 ack_state(msdata); 207 } 208 } while (curstate != MULTI_STOP_EXIT); 0xffffffff810f5714 <+100>: cmp $0x4,%edx 0xffffffff810f5717 <+103>: je 0xffffffff810f5756 0xffffffff810f5751 <+161>: cmp $0x4,%edx 0xffffffff810f5754 <+164>: jne 0xffffffff810f5719 #### qemu processes (guest vcpu mode) > 11703 1 19 ffff882011835fc0 RU 1.6 7144036 2155968 qemu-system-x86 #4 [ffff882011bc1d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 11753 1 20 ffff881811efafe0 RU 1.6 6908396 2128020 qemu-system-x86 #4 [ffff8818126d5d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 11804 1 6 ffff881013f097f0 RU 1.6 6906788 2193896 qemu-system-x86 #4 [ffff881013133d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 11805 1 8 ffff881013f0c7d0 RU 1.6 6906788 2193896 qemu-system-x86 #4 [ffff8810132bfd18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12281 1 17 ffff8820127917f0 RU 1.6 6924620 2194736 qemu-system-x86 #4 [ffff881ff4cbfd18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12285 1 5 ffff880812b45fc0 RU 1.6 6870364 2208328 qemu-system-x86 #4 [ffff8807e2483d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12401 1 3 ffff88081083afe0 RU 1.6 6998156 2208396 qemu-system-x86 #4 [ffff8807e2653d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12402 1 15 ffff8800bcc15fc0 RU 1.6 6998156 2208396 qemu-system-x86 #4 [ffff8807df947d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12456 1 0 ffff88180f4e8000 RU 1.6 6898720 2200512 qemu-system-x86 #4 [ffff8818139d7d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 12529 1 1 ffff8818135e5fc0 RU 1.6 7013648 2220724 qemu-system-x86 #4 [ffff8817e5a29d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] > 29930 1 21 ffff88086e328000 RU 3.1 8950448 4200644 qemu-system-x86 #4 [ffff880842349d18] svm_vcpu_run at ffffffffa01a5507 [kvm_amd] #### qemu processes (hit page fault while doing i/o) ** > 17247 1 23 ffff881007055fc0 RU 1.6 7358428 2192548 qemu-system-x86 PID: 17247 TASK: ffff881007055fc0 CPU: 23 COMMAND: "qemu-system-x86" #0 [ffff88203eac6e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88203eac6e68] nmi_handle at ffffffff8171f188 #2 [ffff88203eac6ec8] do_nmi at ffffffff8171f350 #3 [ffff88203eac6ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: generic_exec_single+130] RIP: ffffffff810db712 RSP: ffff8810ea7c96e0 RFLAGS: 00000202 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000202 RDX: ffff8810ea7c96e0 RSI: 0000000000000018 RDI: 0000000000000001 RBP: ffffffff810db712 R8: ffffffff810db712 R9: 0000000000000018 R10: ffff8810ea7c96e0 R11: 0000000000000202 R12: ffffffffffffffff R13: 0000000000000206 R14: 000000007bc87bc6 R15: ffff8814959f76c0 ORIG_RAX: ffff8814959f76c0 CS: 0010 SS: 0018 --- --- #4 [ffff8810ea7c96e0] generic_exec_single at ffffffff810db712 !!!! CSD_FLAG logic discussed with Linus 108 while (csd->flags & CSD_FLAG_LOCK) 0xffffffff810db712 <+130>: testb $0x1,0x20(%rbx) 0xffffffff810db716 <+134>: jne 0xffffffff810db710 109 cpu_relax(); 110 } #5 [ffff8810ea7c9718] smp_call_function_single at ffffffff810db825 #6 [ffff8810ea7c9790] stop_two_cpus at ffffffff810f5c9f #7 [ffff8810ea7c9880] migrate_swap at ffffffff8109878a #8 [ffff8810ea7c98a8] task_numa_migrate at ffffffff810a07b7 #9 [ffff8810ea7c9980] numa_migrate_preferred at ffffffff810a0ab3 #10 [ffff8810ea7c9990] task_numa_fault at ffffffff810a2360 #11 [ffff8810ea7c9a00] do_numa_page at ffffffff8117763e #12 [ffff8810ea7c9a50] handle_mm_fault at ffffffff811787ff #13 [ffff8810ea7c9ae8] __do_page_fault at ffffffff81721a24 #14 [ffff8810ea7c9be8] do_page_fault at ffffffff81721e1a #15 [ffff8810ea7c9c10] page_fault at ffffffff8171e288 [exception RIP: file_read_actor+71] RIP: ffffffff81150397 RSP: ffff8810ea7c9cc0 RFLAGS: 00010246 RAX: 00007f3ac08ea000 RBX: ffff8810ea7c9d70 RCX: 0000000000000000 RDX: 0000000000000000 RSI: ffffea0079780100 RDI: ffff8810ea7c9d70 RBP: ffff8810ea7c9cf0 R8: 0000000000000002 R9: ffffea007978011c R10: 0000000000000002 R11: 0000000000000000 R12: 0000000000001000 R13: 0000000000001000 R14: ffff882011024f00 R15: 0000000000001000 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 #16 [ffff8810ea7c9cf8] generic_file_aio_read at ffffffff8115078f 1196 prev_index = index; 1197 1198 /* 1199 * Ok, we have the page, and it's up-to-date, so 1200 * now we can copy it to user space... 1201 * 1202 * The file_read_actor routine returns how many bytes were 1203 * actually used.. 1204 * NOTE! This may not be the same as how much of a user buffer 1205 * we filled up (we may be padding etc), so we can only update 1206 * "pos" here (the actor routine has to update the user buffer 1207 * pointers and the remaining count). 1208 */ 1209 ret = file_read_actor(desc, page, offset, nr); 1327 if (PageReadaheadUnused(page)) 0xffffffff81150377 <+39>: test $0x2000000,%eax 0xffffffff8115037c <+44>: mov %rcx,%r12 0xffffffff8115037f <+47>: je 0xffffffff81150386 #17 [ffff8810ea7c9dc0] do_sync_readv_writev at ffffffff811b8e2c #18 [ffff8810ea7c9e40] do_readv_writev at ffffffff811ba2f0 #19 [ffff8810ea7c9f28] vfs_readv at ffffffff811ba48d #20 [ffff8810ea7c9f38] sys_preadv at ffffffff811ba732 #21 [ffff8810ea7c9f80] tracesys at ffffffff817266bf (via system_call) RIP: 00007f3b24f996d2 RSP: 00007f3a8affcb90 RFLAGS: 00000293 RAX: ffffffffffffffda RBX: ffffffff817266bf RCX: ffffffffffffffff RDX: 0000000000000020 RSI: 00007f3b304a0800 RDI: 000000000000000d RBP: 00007f3b304a0800 R8: 0000000000000000 R9: 0000000000000000 R10: 0000000b25a80000 R11: 0000000000000293 R12: 00007f3a8affd700 R13: 00007f3b2c99d438 R14: 0000000b25a80000 R15: 0000000000000020 ORIG_RAX: 0000000000000127 CS: 0033 SS: 002b ** > 21036 1 27 ffff8810b69947d0 RU 1.0 7484828 1401940 qemu-system-x86 PID: 21036 TASK: ffff8810b69947d0 CPU: 27 COMMAND: "qemu-system-x86" #0 [ffff88203eb46e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88203eb46e68] nmi_handle at ffffffff8171f188 #2 [ffff88203eb46ec8] do_nmi at ffffffff8171f350 #3 [ffff88203eb46ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: generic_exec_single+130] RIP: ffffffff810db712 RSP: ffff8814959f7670 RFLAGS: 00000202 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000202 RDX: ffff8814959f7670 RSI: 0000000000000018 RDI: 0000000000000001 RBP: ffffffff810db712 R8: ffffffff810db712 R9: 0000000000000018 R10: ffff8814959f7670 R11: 0000000000000202 R12: ffffffffffffffff R13: 0000000000000282 R14: 0000000000000000 R15: 0000000000000100 ORIG_RAX: 0000000000000100 CS: 0010 SS: 0018 --- --- #4 [ffff8814959f7670] generic_exec_single at ffffffff810db712 !!!! CSD_FLAG logic discussed with Linus 108 while (csd->flags & CSD_FLAG_LOCK) 0xffffffff810db712 <+130>: testb $0x1,0x20(%rbx) 0xffffffff810db716 <+134>: jne 0xffffffff810db710 109 cpu_relax(); 110 } #5 [ffff8814959f76a8] smp_call_function_single at ffffffff810db825 #6 [ffff8814959f7720] stop_two_cpus at ffffffff810f5c9f #7 [ffff8814959f7810] migrate_swap at ffffffff8109878a #8 [ffff8814959f7838] task_numa_migrate at ffffffff810a07b7 #9 [ffff8814959f7910] numa_migrate_preferred at ffffffff810a0ab3 #10 [ffff8814959f7920] task_numa_fault at ffffffff810a2360 #11 [ffff8814959f7990] do_numa_page at ffffffff8117763e #12 [ffff8814959f79e0] handle_mm_fault at ffffffff811787ff #13 [ffff8814959f7a78] __do_page_fault at ffffffff81721a24 #14 [ffff8814959f7b78] do_page_fault at ffffffff81721e1a #15 [ffff8814959f7ba0] page_fault at ffffffff8171e288 [exception RIP: iov_iter_fault_in_readable+49] RIP: ffffffff8114da61 RSP: ffff8814959f7c58 RFLAGS: 00010292 RAX: 0000000000000000 RBX: 000000002df26000 RCX: 0000000000000000 RDX: 00007f085544c000 RSI: 0000000000001000 RDI: ffff8814959f7cc8 RBP: ffff8814959f7c60 R8: ffff8814959f7ec0 R9: 0000000000001000 R10: 000000002df26000 R11: 0000000000001000 R12: 0000000000001000 R13: ffff881007c30200 R14: 0000000000000000 R15: ffff8814959f7e80 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 #16 [ffff8814959f7c68] generic_file_buffered_write at ffffffff8114e30c #17 [ffff8814959f7d18] __generic_file_aio_write at ffffffff8114fa01 #18 [ffff8814959f7d88] generic_file_aio_write at ffffffff8114fc68 #19 [ffff8814959f7dc8] ext4_file_write at ffffffff812366c9 #20 [ffff8814959f7e78] do_sync_write at ffffffff811b8daa #21 [ffff8814959f7ef8] vfs_write at ffffffff811b9534 #22 [ffff8814959f7f38] sys_pwrite64 at ffffffff811ba0e2 #23 [ffff8814959f7f80] tracesys at ffffffff817266bf (via system_call) RIP: 00007f08dd28fd23 RSP: 00007f075dffabf0 RFLAGS: 00000293 RAX: ffffffffffffffda RBX: ffffffff817266bf RCX: ffffffffffffffff RDX: 0000000000001000 RSI: 00007f085544c000 RDI: 000000000000000d RBP: 00007f08e5192a40 R8: 0000000000000000 R9: 0000000000000000 R10: 000000002df26000 R11: 0000000000000293 R12: 00007f08e4ddf438 R13: 00007f08e4ddf498 R14: 00007f085544c000 R15: 0000000000000000 ORIG_RAX: 0000000000000012 CS: 0033 SS: 002b ** > 18516 1 31 ffff881dd54a2fe0 RU 1.6 7358428 2192548 qemu-system-x86 PID: 18516 TASK: ffff881dd54a2fe0 CPU: 31 COMMAND: "qemu-system-x86" #0 [ffff88203ebc6e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88203ebc6e68] nmi_handle at ffffffff8171f188 #2 [ffff88203ebc6ec8] do_nmi at ffffffff8171f350 #3 [ffff88203ebc6ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: generic_exec_single+130] RIP: ffffffff810db712 RSP: ffff881dd55597a0 RFLAGS: 00000202 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000202 RDX: ffff881dd55597a0 RSI: 0000000000000018 RDI: 0000000000000001 RBP: ffffffff810db712 R8: ffffffff810db712 R9: 0000000000000018 R10: ffff881dd55597a0 R11: 0000000000000202 R12: ffffffffffffffff R13: 0000000000000206 R14: 000000007bca7bc8 R15: ffff8814959f76c0 ORIG_RAX: ffff8814959f76c0 CS: 0010 SS: 0018 --- --- #4 [ffff881dd55597a0] generic_exec_single at ffffffff810db712 !!!! CSD_FLAG logic discussed with Linus 108 while (csd->flags & CSD_FLAG_LOCK) 0xffffffff810db712 <+130>: testb $0x1,0x20(%rbx) 0xffffffff810db716 <+134>: jne 0xffffffff810db710 109 cpu_relax(); 110 } #5 [ffff881dd55597d8] smp_call_function_single at ffffffff810db825 #6 [ffff881dd5559850] stop_two_cpus at ffffffff810f5c9f #7 [ffff881dd5559940] migrate_swap at ffffffff8109878a #8 [ffff881dd5559968] task_numa_migrate at ffffffff810a07b7 #9 [ffff881dd5559a40] numa_migrate_preferred at ffffffff810a0ab3 #10 [ffff881dd5559a50] task_numa_fault at ffffffff810a2360 #11 [ffff881dd5559ac0] do_numa_page at ffffffff8117763e #12 [ffff881dd5559b10] handle_mm_fault at ffffffff811787ff #13 [ffff881dd5559ba8] __do_page_fault at ffffffff81721a24 #14 [ffff881dd5559ca8] do_page_fault at ffffffff81721e1a #15 [ffff881dd5559cd0] page_fault at ffffffff8171e288 [exception RIP: file_read_actor+71] RIP: ffffffff81150397 RSP: ffff881dd5559d80 RFLAGS: 00010246 RAX: 00007f3ad938a000 RBX: ffff881dd5559e30 RCX: 0000000000000000 RDX: 0000000000000000 RSI: ffffea0073428c40 RDI: ffff881dd5559e30 RBP: ffff881dd5559db0 R8: 6200000000000000 R9: a801cd0a31000000 R10: 57fe1cf642428c40 R11: 0000000000000f7d R12: 0000000000001000 R13: 0000000000001000 R14: ffff882011024f00 R15: 0000000000001000 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 #16 [ffff881dd5559db8] generic_file_aio_read at ffffffff8115078f #17 [ffff881dd5559e80] do_sync_read at ffffffff811b8d1a #18 [ffff881dd5559f00] vfs_read at ffffffff811b93b5 #19 [ffff881dd5559f38] sys_pread64 at ffffffff811ba032 #20 [ffff881dd5559f80] tracesys at ffffffff817266bf (via system_call) RIP: 00007f3b2527fcc3 RSP: 00007f3a1dffabf0 RFLAGS: 00000293 RAX: ffffffffffffffda RBX: ffffffff817266bf RCX: ffffffffffffffff RDX: 0000000000001000 RSI: 00007f3ad938a000 RDI: 000000000000000d RBP: 00007f3b304a20c0 R8: 0000000000000000 R9: 0000000000000000 R10: 0000000562094000 R11: 0000000000000293 R12: 00007f3b2c99d438 R13: 00007f3b2c99d498 R14: 00007f3ad938a000 R15: 0000000000000000 ORIG_RAX: 0000000000000011 CS: 0033 SS: 002b ** > 21104 1 16 ffff8801478617f0 RU 0.0 304796 2172 qemu-nbd crash> bt ffff8801478617f0 PID: 21104 TASK: ffff8801478617f0 CPU: 16 COMMAND: "qemu-nbd" #0 [ffff88101fc06e58] crash_nmi_callback at ffffffff8103fb72 #1 [ffff88101fc06e68] nmi_handle at ffffffff8171f188 #2 [ffff88101fc06ec8] do_nmi at ffffffff8171f350 #3 [ffff88101fc06ef0] end_repeat_nmi at ffffffff8171e5f1 [exception RIP: io_serial_in+21] RIP: ffffffff81463f35 RSP: ffff88014feb38e8 RFLAGS: 00000002 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000002 RDX: ffff88014feb38e8 RSI: 0000000000000018 RDI: 0000000000000001 RBP: ffffffff81463f35 R8: ffffffff81463f35 R9: 0000000000000018 R10: ffff88014feb38e8 R11: 0000000000000002 R12: ffffffffffffffff R13: ffffffff81fc25a8 R14: 00000000000002fd R15: 00000000000002fd ORIG_RAX: 00000000000002fd CS: 0010 SS: 0018 --- --- #4 [ffff88014feb38e8] io_serial_in at ffffffff81463f35 #5 [ffff88014feb38f0] wait_for_xmitr at ffffffff81464adb #6 [ffff88014feb3918] serial8250_console_putchar at ffffffff81464c9c #7 [ffff88014feb3938] uart_console_write at ffffffff8145f8da #8 [ffff88014feb3968] serial8250_console_write at ffffffff81464bee #9 [ffff88014feb39b0] call_console_drivers.constprop.19 at ffffffff810bc033 #10 [ffff88014feb39d8] console_unlock at ffffffff810bd19f #11 [ffff88014feb3a10] vprintk_emit at ffffffff810bd49e #12 [ffff88014feb3a80] dev_vprintk_emit at ffffffff8148a782 #13 [ffff88014feb3b38] dev_printk_emit at ffffffff8148a7ee #14 [ffff88014feb3b90] __dev_printk at ffffffff8148ac0e #15 [ffff88014feb3ba0] dev_err at ffffffff8148ae2c #16 [ffff88014feb3c08] __nbd_ioctl at ffffffffa01fd45c [nbd] #17 [ffff88014feb3e30] nbd_ioctl at ffffffffa01fd4fa [nbd] #18 [ffff88014feb3e68] blkdev_ioctl at ffffffff8133f84c #19 [ffff88014feb3eb8] block_ioctl at ffffffff811f26c1 #20 [ffff88014feb3ec8] do_vfs_ioctl at ffffffff811cc6e0 #21 [ffff88014feb3f38] sys_ioctl at ffffffff811cc941 #22 [ffff88014feb3f80] tracesys at ffffffff817266bf (via system_call) RIP: 00007f4d887a9337 RSP: 00007f4d821dcdf8 RFLAGS: 00000246 RAX: ffffffffffffffda RBX: ffffffff817266bf RCX: ffffffffffffffff RDX: 00007f4d819dc9d0 RSI: 000000000000ab03 RDI: 000000000000000c RBP: 00007f4d821dce60 R8: 00007f4d819dc700 R9: 00007f4d819dc700 R10: 00007f4d819dc9d0 R11: 0000000000000246 R12: 00007f4d821dd700 R13: 00007f4d821dd9c0 R14: 00007fff74966e33 R15: 000000000000000c ORIG_RAX: 0000000000000010 CS: 0033 SS: 002b