I think this patch fixes the issue:
https://lkml.org/lkml/2014/9/22/211
Looking at the stacktrace:
[ 4.690909] divide error: 0000 [#1] SMP [ 4.690909] Modules linked in: dm_crypt kvm_amd kvm serio_raw isofs crct10dif_pclmul crc32_pclmul ghash_clmulni_intel aesni_intel aes_x86_64 lrw gf128mul glue_helper ablk_helper cryptd psmouse floppy [ 4.690909] CPU: 0 PID: 663 Comm: cloud-init Not tainted 3.13.0-40-generic #69-Ubuntu [ 4.690909] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011 [ 4.690909] task: ffff88001f373000 ti: ffff88001460a000 task.ti: ffff88001460a000 [ 4.690909] RIP: 0010:[<ffffffff8104ed58>] [<ffffffff8104ed58>] kvm_unlock_kick+0xa8/0x100 [ 4.690909] RSP: 0000:ffff88001fc03df0 EFLAGS: 00010046 [ 4.690909] RAX: 0000000000000005 RBX: 0000000000000000 RCX: 0000000000000001 [ 4.690909] RDX: ffffffff81eb1448 RSI: 0000000000000000 RDI: 0000000000000000 [ 4.690909] RBP: ffff88001fc03e10 R08: ffffffff81eb1440 R09: ffff880016000000 [ 4.690909] R10: 0000000000000006 R11: 561488f3089a6867 R12: ffffffff81fc66c0 [ 4.690909] R13: 0000000000000802 R14: 0000000000000001 R15: 00000000000000c2 [ 4.690909] FS: 00007fc269f46740(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000 [ 4.690909] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4.690909] CR2: 00007fc2665de050 CR3: 000000001f50f000 CR4: 00000000000406f0 [ 4.690909] Stack: [ 4.690909] 0000000000000046 0000000000000060 0000000000000046 0000000000000020 [ 4.690909] ffff88001fc03e20 ffffffff81718b53 ffff88001fc03e38 ffffffff817270da [ 4.690909] ffffffff81fc66c0 ffff88001fc03e70 ffffffff8146de04 ffffffff81fc66c0 [ 4.690909] Call Trace: [ 4.690909] <IRQ> [ 4.690909] [<ffffffff81718b53>] __ticket_unlock_slowpath+0x24/0x34 [ 4.690909] [<ffffffff817270da>] _raw_spin_unlock_irqrestore+0x3a/0x40 [ 4.690909] [<ffffffff8146de04>] serial8250_handle_irq.part.14+0x84/0xb0 [ 4.690909] [<ffffffff8146de77>] serial8250_default_handle_irq+0x27/0x30 [ 4.690909] [<ffffffff8146ce73>] serial8250_interrupt+0x63/0xe0 [ 4.690909] [<ffffffff810bf97e>] handle_irq_event_percpu+0x3e/0x1d0 [ 4.690909] [<ffffffff810bfb4d>] handle_irq_event+0x3d/0x60 [ 4.690909] [<ffffffff810c25d7>] handle_edge_irq+0x77/0x130 [ 4.690909] [<ffffffff81015dbe>] handle_irq+0x1e/0x30 [ 4.690909] [<ffffffff8173205d>] do_IRQ+0x4d/0xc0 [ 4.690909] [<ffffffff8172772d>] common_interrupt+0x6d/0x6d [ 4.690909] <EOI> [ 4.690909] Code: 66 44 39 e8 75 bd 0f b6 35 36 27 e6 00 40 84 f6 75 2a 83 05 46 27 e6 00 01 48 c7 c0 8a b0 00 00 31 db 0f b7 0c 01 b8 05 00 00 00 <0f> 01 c1 0f 1f 44 00 00 5b 41 5c 41 5d 41 5e 5d c3 89 f0 31 c9 [ 4.690909] RIP [<ffffffff8104ed58>] kvm_unlock_kick+0xa8/0x100 [ 4.690909] RSP <ffff88001fc03df0>
Looking at the objdump we see we get a Divide Error on a vmcall instruction. In addition we build our kernels with CONFIG_DEBUG_RODATA and PV locking.
static void kvm_kick_cpu(int cpu) { int apicid; unsigned long flags = 0;
apicid = per_cpu(x86_cpu_to_apicid, cpu); ffffffff8104ed46: 48 c7 c0 8a b0 00 00 mov $0xb08a,%rax
static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, unsigned long p2) { long ret; asm volatile(KVM_HYPERCALL ffffffff8104ed4d: 31 db xor %ebx,%ebx kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid); ffffffff8104ed4f: 0f b7 0c 01 movzwl (%rcx,%rax,1),%ecx ffffffff8104ed53: b8 05 00 00 00 mov $0x5,%eax ffffffff8104ed58: 0f 01 c1 vmcall ffffffff8104ed5b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) add_stats(RELEASED_SLOW_KICKED, 1); kvm_kick_cpu(cpu); break; } } }
I think this patch fixes the issue:
https:/ /lkml.org/ lkml/2014/ 9/22/211
Looking at the stacktrace:
[ 4.690909] divide error: 0000 [#1] SMP ffffffff8104ed5 8>] [<ffffffff8104e d58>] kvm_unlock_ kick+0xa8/ 0x100 c03df0 EFLAGS: 00010046 0(0000) GS:ffff88001fc0 0000(0000) knlGS:000000000 0000000 b53>] __ticket_ unlock_ slowpath+ 0x24/0x34 0da>] _raw_spin_ unlock_ irqrestore+ 0x3a/0x40 e04>] serial8250_ handle_ irq.part. 14+0x84/ 0xb0 e77>] serial8250_ default_ handle_ irq+0x27/ 0x30 e73>] serial8250_ interrupt+ 0x63/0xe0 97e>] handle_ irq_event_ percpu+ 0x3e/0x1d0 b4d>] handle_ irq_event+ 0x3d/0x60 5d7>] handle_ edge_irq+ 0x77/0x130 dbe>] handle_ irq+0x1e/ 0x30 05d>] do_IRQ+0x4d/0xc0 72d>] common_ interrupt+ 0x6d/0x6d d58>] kvm_unlock_ kick+0xa8/ 0x100
[ 4.690909] Modules linked in: dm_crypt kvm_amd kvm serio_raw isofs crct10dif_pclmul crc32_pclmul ghash_clmulni_intel aesni_intel aes_x86_64 lrw gf128mul glue_helper ablk_helper cryptd psmouse floppy
[ 4.690909] CPU: 0 PID: 663 Comm: cloud-init Not tainted 3.13.0-40-generic #69-Ubuntu
[ 4.690909] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
[ 4.690909] task: ffff88001f373000 ti: ffff88001460a000 task.ti: ffff88001460a000
[ 4.690909] RIP: 0010:[<
[ 4.690909] RSP: 0000:ffff88001f
[ 4.690909] RAX: 0000000000000005 RBX: 0000000000000000 RCX: 0000000000000001
[ 4.690909] RDX: ffffffff81eb1448 RSI: 0000000000000000 RDI: 0000000000000000
[ 4.690909] RBP: ffff88001fc03e10 R08: ffffffff81eb1440 R09: ffff880016000000
[ 4.690909] R10: 0000000000000006 R11: 561488f3089a6867 R12: ffffffff81fc66c0
[ 4.690909] R13: 0000000000000802 R14: 0000000000000001 R15: 00000000000000c2
[ 4.690909] FS: 00007fc269f4674
[ 4.690909] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 4.690909] CR2: 00007fc2665de050 CR3: 000000001f50f000 CR4: 00000000000406f0
[ 4.690909] Stack:
[ 4.690909] 0000000000000046 0000000000000060 0000000000000046 0000000000000020
[ 4.690909] ffff88001fc03e20 ffffffff81718b53 ffff88001fc03e38 ffffffff817270da
[ 4.690909] ffffffff81fc66c0 ffff88001fc03e70 ffffffff8146de04 ffffffff81fc66c0
[ 4.690909] Call Trace:
[ 4.690909] <IRQ>
[ 4.690909] [<ffffffff81718
[ 4.690909] [<ffffffff81727
[ 4.690909] [<ffffffff8146d
[ 4.690909] [<ffffffff8146d
[ 4.690909] [<ffffffff8146c
[ 4.690909] [<ffffffff810bf
[ 4.690909] [<ffffffff810bf
[ 4.690909] [<ffffffff810c2
[ 4.690909] [<ffffffff81015
[ 4.690909] [<ffffffff81732
[ 4.690909] [<ffffffff81727
[ 4.690909] <EOI>
[ 4.690909] Code: 66 44 39 e8 75 bd 0f b6 35 36 27 e6 00 40 84 f6 75 2a 83 05 46 27 e6 00 01 48 c7 c0 8a b0 00 00 31 db 0f b7 0c 01 b8 05 00 00 00 <0f> 01 c1 0f 1f 44 00 00 5b 41 5c 41 5d 41 5e 5d c3 89 f0 31 c9
[ 4.690909] RIP [<ffffffff8104e
[ 4.690909] RSP <ffff88001fc03df0>
Looking at the objdump we see we get a Divide Error on a vmcall instruction.
In addition we build our kernels with CONFIG_DEBUG_RODATA and PV locking.
static void kvm_kick_cpu(int cpu)
{
int apicid;
unsigned long flags = 0;
apicid = per_cpu( x86_cpu_ to_apicid, cpu);
ffffffff8104ed46: 48 c7 c0 8a b0 00 00 mov $0xb08a,%rax
static inline long kvm_hypercall2( unsigned int nr, unsigned long p1,
unsigned long p2) KVM_HYPERCALL
kvm_hypercall2 (KVM_HC_ KICK_CPU, flags, apicid);
add_ stats(RELEASED_ SLOW_KICKED, 1);
kvm_ kick_cpu( cpu);
break;
{
long ret;
asm volatile(
ffffffff8104ed4d: 31 db xor %ebx,%ebx
ffffffff8104ed4f: 0f b7 0c 01 movzwl (%rcx,%rax,1),%ecx
ffffffff8104ed53: b8 05 00 00 00 mov $0x5,%eax
ffffffff8104ed58: 0f 01 c1 vmcall
ffffffff8104ed5b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
}
}
}