We've observed this on another instance running 12.04, ami-3c994355, with a read-heavy postgresql load. CPU0 rip: ffffffff8105e51a try_to_wake_up+0xca flags: 00001202 i nz rsp: ffff880f22deb7d0 rax: 0000000000000004 rcx: ffff880f22deb900 rdx: 0000000000000082 rbx: ffff880c633196e0 rsi: 0000000000000000 rdi: ffff880c63319cc0 rbp: ffff880f22deb820 r8: ffffea0024739ec0 r9: 0000000000000001 r10: 57ff5831a9739ec0 r11: 0000000000000000 r12: ffff880c63319cc0 r13: 0000000000000000 r14: 0000000000000000 r15: 0000000000000003 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22de8000/0000000000000000 Code (instr addr ffffffff8105e51a) 66 90 45 85 ff 74 0e 44 8b 7d cc e9 aa 00 00 00 0f 1f 00 f3 90 <44> 8b 4b 28 45 85 c9 75 f5 48 8b Stack: ffffffff81c70004 ffffffff81c70004 0000000000000001 000000041e7b3969 0000000000000000 ffff880a931f9c70 0000000000000001 ffff880f22fbcaf0 ffff880f22deb900 0000000000000000 ffff880f22deb830 ffffffff8105e662 ffff880f22deb850 ffffffff81089356 0000007fff570000 ffff880f22fbcad8 Call Trace: [] try_to_wake_up+0xca <-- [] default_wake_function+0x12 [] autoremove_wake_function+0x16 [] wake_bit_function+0x3b [] __wake_up_common+0x58 [] __wake_up+0x48 [] __wake_up_bit+0x31 [] unlock_page+0x2a [] mpage_end_io+0x46 [] bio_endio+0x1d [] req_bio_endio.isra.45+0xa3 [] blk_update_request+0xf5 [] blk_update_bidi_request+0x31 [] __blk_end_bidi_request+0x20 [] __blk_end_request_all+0x1f [] blkif_interrupt+0x1cc [] handle_irq_event_percpu+0x55 [] radix_tree_lookup+0xb [] handle_irq_event+0x4e [] handle_edge_irq+0x84 [] __xen_evtchn_do_upcall+0x199 [] xen_evtchn_do_upcall+0x2f [] xen_do_hypervisor_callback+0x1e [] hypercall_page+0x3aa [] hypercall_page+0x3aa [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock_flags+0x63 [] _raw_spin_lock_irqsave+0x2e [] update_shares+0x9e [] rebalance_domains+0x48 [] run_rebalance_domains+0x48 [] __do_softirq+0xa8 [] __xen_evtchn_do_upcall+0x207 [] call_softirq+0x1c [] do_softirq+0x65 [] irq_exit+0x8e [] xen_evtchn_do_upcall+0x35 [] xen_do_hypervisor_callback+0x1e CPU1 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001246 i z p rsp: ffff880ec1225ec8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: ffff880ec1225fd8 rsi: 0000000000000000 rdi: 0000000000000001 rbp: ffff880ec1225ee0 r8: 0000000000000000 r9: 0000000000000000 r10: 0000000000000001 r11: 0000000000000246 r12: ffffffff81cdbd60 r13: 0000000000000001 r14: 0000000000000000 r15: 0000000000000000 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22e04000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000000 00000000ffffffff ffffffff8100a190 ffff880ec1225f10 ffffffff8101b893 ffff880ec1225fd8 ffffffff81cdbd60 0000000000000000 0000000000000000 ffff880ec1225f40 ffffffff81012236 ffffffff8100a9d9 58e9d471e076535d 0000000000000000 0000000000000000 ffff880ec1225f50 Call Trace: [] hypercall_page+0x3aa <-- [] xen_safe_halt+0x10 [] default_idle+0x53 [] cpu_idle+0xd6 [] xen_irq_enable_direct_reloc+0x4 [] cpu_bringup_and_idle+0xe CPU2 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880ec0ef1930 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec0ef1948 rdi: 0000000000000003 rbp: ffff880ec0ef1978 r8: ffff880ec316d700 r9: ffff880ee5400100 r10: 0000000000000022 r11: 0000000000000202 r12: 000000000000001d r13: 0000000000000001 r14: ffff880ec0ef1a01 r15: ffff880c63319c00 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f7566bae700 gs: 0000 @ ffff880f22e20000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000246 00000000fffffffa ffffffff813a464e ffff880ec0ef1964 ffffffff00000001 0000000000000000 00000010813a404e ffff880ec0ef1978 ffff880c63319cc0 ffff880ec0ef1988 ffffffff813a6130 ffff880ec0ef19d8 ffffffff8163a7a1 0000000200000000 0000000000000000 000000000000ffff Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock_flags+0x63 [] _raw_spin_lock_irqsave+0x2e [] task_rq_lock+0x40 [] task_sched_runtime+0x2c [] thread_group_cputime+0x78 [] thread_group_times+0x33 [] do_task_stat+0x6c3 [] mem_cgroup_add_lru_list+0x1a [] xen_set_pte_at+0x39 [] pte_mfn_to_pfn+0x89 [] xen_pte_val+0x30 [] __raw_callee_save_xen_pte_val+0x11 [] unlock_page+0x2a [] follow_page+0x322 [] sys_mincore+0x131 [] proc_tgid_stat+0x14 [] proc_single_show+0x5c [] seq_read+0x123 [] vfs_read+0xb0 [] sys_read+0x4a [] system_call_fastpath+0x16 CPU3 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880c50209950 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880c50209968 rdi: 0000000000000003 rbp: ffff880c50209998 r8: ffff880ec316dd00 r9: ffff880ee5400130 r10: 0000000000000000 r11: 0000000000000202 r12: 0000000000000023 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22e4f600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22e3c000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880c50209984 ffffffff00000001 0000000000000000 00000016813a404e ffff880c50209998 ffff880f22dfb6c0 ffff880c502099a8 ffffffff813a6130 ffff880c502099f8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] sleep_on_page+0x20 [] __schedule+0x699 [] sleep_on_page+0x20 [] schedule+0x3f [] io_schedule+0x8f [] sleep_on_page_killable+0xe [] __wait_on_bit_lock+0x5a [] __lock_page_killable+0x67 [] pte_pfn_to_mfn+0x88 [] autoremove_wake_function+0x40 [] do_generic_file_read.constprop.33+0x287 [] generic_file_aio_read+0xef [] handle_pte_fault+0xfa [] do_sync_read+0xd2 [] security_file_permission+0x93 [] rw_verify_area+0x61 [] vfs_read+0xb0 [] sys_read+0x4a [] system_call_fastpath+0x16 CPU4 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880a931f9950 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880a931f9968 rdi: 0000000000000003 rbp: ffff880a931f9998 r8: ffff880ec12fe300 r9: ffff880ee5400160 r10: 0000000000000000 r11: 0000000000000202 r12: 0000000000000029 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22e6b600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22e58000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880a931f9984 ffffffff00000001 0000000000000000 0000001c813a404e ffff880a931f9998 ffff880f22dfb6c0 ffff880a931f99a8 ffffffff813a6130 ffff880a931f99f8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] sleep_on_page+0x20 [] __schedule+0x699 [] sleep_on_page+0x20 [] schedule+0x3f [] io_schedule+0x8f [] sleep_on_page_killable+0xe [] __wait_on_bit_lock+0x5a [] __lock_page_killable+0x67 [] pte_pfn_to_mfn+0x88 [] autoremove_wake_function+0x40 [] do_generic_file_read.constprop.33+0x287 [] generic_file_aio_read+0xef [] handle_pte_fault+0xfa [] do_sync_read+0xd2 [] security_file_permission+0x93 [] rw_verify_area+0x61 [] vfs_read+0xb0 [] sys_read+0x4a [] system_call_fastpath+0x16 CPU5 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880f22e77c20 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880f22e77c38 rdi: 0000000000000003 rbp: ffff880f22e77c68 r8: ffff880ec12fe900 r9: ffff880ee5400190 r10: 0000000000000001 r11: 0000000000000206 r12: 000000000000002f r13: 0000000000000001 r14: ffffffff81e0b101 r15: 7fffffffffffff00 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007ff5b1375700 gs: 0000 @ ffff880f22e74000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000000 00000000fffffffa ffffffff813a464e ffff880f22e77c54 ffffffff00000001 0000000000000000 00000022813a404e ffff880f22e77c68 ffff880f22dfb6c0 ffff880f22e77c78 ffffffff813a6130 ffff880f22e77cc8 ffffffff8163a7a1 0000000000000005 0000000000000000 ffff880f22e77ca8 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] do_sched_rt_period_timer+0x138 [] xen_clocksource_read+0x20 [] sched_rt_period_timer+0x2a [] __run_hrtimer+0x78 [] do_sched_rt_period_timer+0x370 [] xen_clocksource_get_cycles+0x9 [] hrtimer_interrupt+0xe3 [] xen_timer_interrupt+0x27 [] handle_irq_event_percpu+0x55 [] info_for_irq+0xe [] handle_percpu_irq+0x47 [] __xen_evtchn_do_upcall+0x199 [] xen_evtchn_do_upcall+0x2f [] xen_do_hypervisor_callback+0x1e CPU6 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880ebd6c7780 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ebd6c7798 rdi: 0000000000000003 rbp: ffff880ebd6c77c8 r8: ffff880ec12fef00 r9: ffff880ee54001c0 r10: 0000000000000000 r11: 0000000000000206 r12: 0000000000000035 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22ea3600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22e90000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880ebd6c77b4 ffffffff00000001 0000000000000000 00000028813a404e ffff880ebd6c77c8 ffff880f22dfb6c0 ffff880ebd6c77d8 ffffffff813a6130 ffff880ebd6c7828 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] _raw_spin_unlock_irqrestore+0x1e [] __schedule+0x699 [] schedule+0x3f [] schedule_hrtimeout_range_clock+0x12c [] update_rmtp+0x70 [] hrtimer_start_range_ns+0x14 [] schedule_hrtimeout_range+0x13 [] poll_schedule_timeout+0x49 [] do_poll.isra.4+0x1ca [] do_sys_poll+0x1b9 [] poll_freewait+0xe0 [] __pollwait+0xf0 [] inet_recvmsg+0x6b [] sock_recvmsg+0x11d [] xen_set_pte_at+0x39 [] do_anonymous_page.isra.38+0x247 [] handle_pte_fault+0x1e1 [] xen_pmd_val+0xe [] __raw_callee_save_xen_pmd_val+0x11 [] handle_mm_fault+0x1f8 [] pvclock_clocksource_read+0x55 [] xen_force_evtchn_callback+0xd [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] ktime_get_ts+0xad [] poll_select_set_timeout+0x7a [] sys_poll+0x76 [] system_call_fastpath+0x16 CPU7 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880c98f8fb60 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880c98f8fb78 rdi: 0000000000000003 rbp: ffff880c98f8fba8 r8: ffff880ec12ff500 r9: ffff880ee54001f0 r10: 57fefef53fc2bec0 r11: 0000000000000206 r12: 000000000000003b r13: 0000000000000001 r14: 0000000000000001 r15: ffff880c98f8fc01 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22eac000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: ffffea003ac2bec0 00000000fffffffa ffffffff813a464e ffff880c98f8fb94 ffffffff00000001 0000000000000000 0000002e813a404e ffff880c98f8fba8 ffff880f22fbcae8 ffff880c98f8fbb8 ffffffff813a6130 ffff880c98f8fc08 ffffffff8163a7a1 ffff880c98f8fbd8 0000000000000000 ffff880e653e73f0 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock_flags+0x63 [] _raw_spin_lock_irqsave+0x2e [] __wake_up+0x32 [] __wake_up_bit+0x31 [] unlock_page+0x2a [] __do_fault+0x439 [] mntput+0x1f [] handle_pte_fault+0xfa [] xen_pmd_val+0xe [] __raw_callee_save_xen_pmd_val+0x11 [] handle_mm_fault+0x1f8 [] do_page_fault+0x14b [] strncpy_from_user+0x31 [] kmem_cache_free+0x2f [] ext4_llseek+0x2c [] page_fault+0x25 CPU8 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880bf12f7840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880bf12f7858 rdi: 0000000000000003 rbp: ffff880bf12f7888 r8: ffff880ec12ffb00 r9: ffff880ee5400490 r10: 0000000000000000 r11: 0000000000000202 r12: 0000000000000041 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22edb600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22ec8000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880bf12f7874 ffffffff00000001 0000000000000000 00000034813a404e ffff880bf12f7888 ffff880f22dfb6c0 ffff880bf12f7898 ffffffff813a6130 ffff880bf12f78e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] xen_force_evtchn_callback+0xd [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] sys_setitimer+0x61 [] system_call_fastpath+0x16 CPU9 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880c96e9d840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880c96e9d858 rdi: 0000000000000003 rbp: ffff880c96e9d888 r8: ffff880ec13a2100 r9: ffff880ee54004c0 r10: 0000000000000000 r11: 0000000000000206 r12: 0000000000000047 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22ef7600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22ee4000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880c96e9d874 ffffffff00000001 0000000000000000 0000003a813a404e ffff880c96e9d888 ffff880f22dfb6c0 ffff880c96e9d898 ffffffff813a6130 ffff880c96e9d8e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] error_exit+0x2a [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] generic_file_aio_read+0xef [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] getnstimeofday+0x57 [] do_gettimeofday+0x1a [] system_call_fastpath+0x16 CPU10 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880e6e867840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880e6e867858 rdi: 0000000000000003 rbp: ffff880e6e867888 r8: ffff880ec13a2700 r9: ffff880ee54004f0 r10: 0000000000000000 r11: 0000000000000202 r12: 000000000000004d r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22f13600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f00000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880e6e867874 ffffffff00000001 0000000000000000 00000040813a404e ffff880e6e867888 ffff880f22dfb6c0 ffff880e6e867898 ffffffff813a6130 ffff880e6e8678e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] xen_force_evtchn_callback+0xd [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] getnstimeofday+0x57 [] do_gettimeofday+0x1a [] system_call_fastpath+0x16 CPU11 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880b5e2bf840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880b5e2bf858 rdi: 0000000000000003 rbp: ffff880b5e2bf888 r8: ffff880ec13a2d00 r9: ffff880ee5400520 r10: 0000000000000000 r11: 0000000000000202 r12: 0000000000000053 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22f2f600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f1c000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880b5e2bf874 ffffffff00000001 0000000000000000 00000046813a404e ffff880b5e2bf888 ffff880f22dfb6c0 ffff880b5e2bf898 ffffffff813a6130 ffff880b5e2bf8e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] xen_force_evtchn_callback+0xd [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] error_exit+0x2a [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] generic_file_aio_read+0xef [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] getnstimeofday+0x57 [] do_gettimeofday+0x1a [] system_call_fastpath+0x16 CPU12 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880c3b303840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880c3b303858 rdi: 0000000000000003 rbp: ffff880c3b303888 r8: ffff880ec13a3300 r9: ffff880ee5400550 r10: 0000000000000000 r11: 0000000000000202 r12: 0000000000000059 r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22f4b600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f38000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880c3b303874 ffffffff00000001 0000000000000000 0000004c813a404e ffff880c3b303888 ffff880f22dfb6c0 ffff880c3b303898 ffffffff813a6130 ffff880c3b3038e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] do_anonymous_page.isra.38+0x247 [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] getnstimeofday+0x57 [] do_gettimeofday+0x1a [] system_call_fastpath+0x16 CPU13 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880f22f57cb8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880f22f57cd0 rdi: 0000000000000003 rbp: ffff880f22f57d00 r8: ffff880ec13a3900 r9: ffff880ee5400580 r10: 0000000000000000 r11: 0000000000000202 r12: 000000000000005f r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22f67600 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f54000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880f22f57cec ffffffff00000001 0000000000000000 00000052813a404e ffff880f22f57d00 ffff880f22dfb6c0 ffff880f22f57d10 ffffffff813a6130 ffff880f22f57d60 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] rebalance_domains+0x99 [] xen_timer_interrupt+0x2c [] run_rebalance_domains+0x48 [] __do_softirq+0xa8 [] __xen_evtchn_do_upcall+0x207 [] call_softirq+0x1c [] do_softirq+0x65 [] irq_exit+0x8e [] xen_evtchn_do_upcall+0x35 [] xen_do_hypervisor_callback+0x1e CPU14 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880b45eb3800 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880b45eb3818 rdi: 0000000000000003 rbp: ffff880b45eb3848 r8: ffff880ec13a3f00 r9: ffff880ee54005b0 r10: 0000000015091c00 r11: 0000000000000202 r12: 0000000000000065 r13: 0000000000000001 r14: ffff880eab073501 r15: ffff880ebe3ee300 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f70000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000000 00000000fffffffa ffffffff813a464e ffff880b45eb3834 ffffffff00000001 0000000000000000 00000058813a404e ffff880b45eb3848 ffffffff81f15eb0 ffff880b45eb3858 ffffffff813a6130 ffff880b45eb38a8 ffffffff8163a7a1 ffff880ebe3ee3c0 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock_irq+0x15 [] blk_queue_bio+0x154 [] generic_make_request.part.50+0x74 [] generic_make_request+0x68 [] submit_bio+0x85 [] add_to_page_cache_locked+0x85 [] mpage_readpages+0x120 [] noalloc_get_block_write+0x30 [] __alloc_pages_nodemask+0x109 [] noalloc_get_block_write+0x30 [] xen_force_evtchn_callback+0xd [] ext4_readpages+0x1d [] read_pages+0x48 [] __do_page_cache_readahead+0x163 [] radix_tree_lookup+0xb [] ondemand_readahead+0x189 [] pte_pfn_to_mfn+0x88 [] page_cache_sync_readahead+0x31 [] do_generic_file_read.constprop.33+0x2ae [] generic_file_aio_read+0xef [] handle_pte_fault+0xfa [] do_sync_read+0xd2 [] security_file_permission+0x93 [] rw_verify_area+0x61 [] vfs_read+0xb0 [] sys_read+0x4a [] system_call_fastpath+0x16 CPU15 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880c49ec9840 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880c49ec9858 rdi: 0000000000000003 rbp: ffff880c49ec9888 r8: ffff880ec0c40500 r9: ffff880ee54005e0 r10: 0000000000000000 r11: 0000000000000202 r12: 000000000000006b r13: 0000000000000001 r14: ffff880f22dfb601 r15: ffff880f22f9f600 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007ff790365740 gs: 0000 @ ffff880f22f8c000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff813a464e ffff880c49ec9874 ffffffff00000001 0000000000000000 0000005e813a404e ffff880c49ec9888 ffff880f22dfb6c0 ffff880c49ec9898 ffffffff813a6130 ffff880c49ec98e8 ffffffff8163a7a1 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] hypercall_page+0x3aa <-- [] xen_poll_irq_timeout+0x3e [] xen_poll_irq+0x10 [] xen_spin_lock_slow+0x97 [] xen_spin_lock+0x4a [] _raw_spin_lock+0xe [] double_rq_lock+0x2c [] load_balance+0xf8 [] _raw_spin_unlock_irqrestore+0x1e [] idle_balance+0xab [] __schedule+0x699 [] xen_force_evtchn_callback+0xd [] schedule+0x3f [] schedule_timeout+0x2a5 [] xen_restore_fl_direct_reloc+0x4 [] sk_wait_data+0xd1 [] add_wait_queue+0x60 [] tcp_recvmsg+0x4c8 [] inet_recvmsg+0x6b [] inet_sendmsg+0x64 [] sock_recvmsg+0x11d [] sock_sendmsg+0x10e [] xen_force_evtchn_callback+0xd [] check_events+0x12 [] check_events+0x12 [] sys_recvfrom+0xee [] xen_clocksource_read+0x20 [] xen_clocksource_get_cycles+0x9 [] getnstimeofday+0x57 [] do_gettimeofday+0x1a [] system_call_fastpath+0x16