BUG: unable to handle kernel paging request at ee835a95

[Juerg Haefliger]

Booting Bionic i386 in a VM and running the following commands repeatedly:

ovs-vsctl add-br test
ovs-vsctl del-br test

eventually leads to:

[ 44.476751] IP: kmem_cache_alloc_trace+0x91/0x1d0
[ 44.477299] *pdpt = 000000001ae13001 *pde = 0000000000000000
[ 44.477956] Oops: 0000 [#1] SMP
[ 44.478340] Modules linked in: dummy openvswitch nsh nf_conntrack_ipv6 nf_nat_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_defrag_ipv6 nf_nat nf_conntrack isofs kvm_intel kvm irqbypass input_leds joydev serio_raw sch_fq_codel ib_iser rdma_cm iw_cm ib_cm ib_core iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi ip_tables x_tables autofs4 btrfs zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear psmouse virtio_blk virtio_net floppy
[ 44.483687] CPU: 0 PID: 553 Comm: systemd-network Tainted: G W 4.15.0-44-generic #47-Ubuntu
[ 44.484819] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.11.1-1ubuntu1 04/01/2014
[ 44.485865] EIP: kmem_cache_alloc_trace+0x91/0x1d0
[ 44.486451] EFLAGS: 00010286 CPU: 0
[ 44.486917] EAX: df719701 EBX: ee835a95 ECX: 0000e8e4 EDX: 0000e8e3
[ 44.487663] ESI: df7197e0 EDI: df401a00 EBP: df7cbda0 ESP: df7cbd78
[ 44.488440] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[ 44.489103] CR0: 80050033 CR2: ee835a95 CR3: 1f7f7ac0 CR4: 000006f0
[ 44.489918] DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
[ 44.490717] DR6: fffe0ff0 DR7: 00000400
[ 44.491265] Call Trace:
[ 44.491655] ? seq_open+0x2d/0x80
[ 44.492136] seq_open+0x2d/0x80
[ 44.492602] kernfs_fop_open+0x1a0/0x360
[ 44.493146] do_dentry_open+0x1ac/0x2f0
[ 44.493688] ? kernfs_seq_start+0x90/0x90
[ 44.494258] vfs_open+0x41/0x70
[ 44.494717] path_openat+0x5e0/0x13f0
[ 44.495234] ? dput.part.23+0xcf/0x1e0
[ 44.495761] ? mntput+0x20/0x40
[ 44.496227] do_filp_open+0x6a/0xd0
[ 44.496739] ? __alloc_fd+0x36/0x160
[ 44.497267] do_sys_open+0x1ad/0x2b0
[ 44.497800] SyS_openat+0x1b/0x20
[ 44.498297] do_fast_syscall_32+0x7f/0x1e0
[ 44.498882] entry_SYSENTER_32+0x4e/0x7c
[ 44.499450] EIP: 0xb7f0dd09
[ 44.499888] EFLAGS: 00000282 CPU: 0
[ 44.500410] EAX: ffffffda EBX: ffffff9c ECX: bfa5fdc0 EDX: 00088000
[ 44.501244] ESI: 00000000 EDI: 00000000 EBP: 00088000 ESP: bfa5fc60
[ 44.502078] DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 007b
[ 44.502833] Code: 33 87 b8 00 00 00 89 75 dc 89 c3 89 45 e0 8b 45 f0 31 f3 8b 37 64 0f c7 0e 0f 94 c0 84 c0 74 bb 8b 75 dc 3b 75 e0 74 0e 03 5f 14 <33> 1b 33 9f b8 00 00 00 0f 18 03 f7 45 ec 00 80 00 00 0f 85 f7
[ 44.505346] EIP: kmem_cache_alloc_trace+0x91/0x1d0 SS:ESP: 0068:df7cbd78
[ 44.506250] CR2: 00000000ee835a95
[ 44.506751] ---[ end trace 3c49b27dd79507a3 ]---
[ 44.508624] BUG: unable to handle kernel paging request at ee835a95
[ 44.509508] IP: __kmalloc+0x85/0x220
[ 44.510020] *pdpt = 000000001ae13001 *pde = 0000000000000000
[ 44.510829] Oops: 0000 [#2] SMP
[ 44.511307] Modules linked in: dummy openvswitch nsh nf_conntrack_ipv6 nf_nat_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_defrag_ipv6 nf_nat nf_conntrack isofs kvm_intel kvm irqbypass input_leds joydev serio_raw sch_fq_codel ib_iser rdma_cm iw_cm ib_cm ib_core iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi ip_tables x_tables autofs4 btrfs zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear psmouse virtio_blk virtio_net floppy
[ 44.517190] CPU: 0 PID: 1854 Comm: journal-offline Tainted: G D W 4.15.0-44-generic #47-Ubuntu
[ 44.518478] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.11.1-1ubuntu1 04/01/2014
[ 44.519730] EIP: __kmalloc+0x85/0x220
[ 44.520275] EFLAGS: 00010086 CPU: 0
[ 44.520794] EAX: dfacdbf0 EBX: 00000000 ECX: ee835a95 EDX: 0000e8e4
[ 44.521666] ESI: dae09bf0 EDI: df401a00 EBP: dcaf59fc ESP: dcaf59d4
[ 44.522497] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[ 44.523215] CR0: 80050033 CR2: ee835a95 CR3: 1cf932a0 CR4: 000006f0
[ 44.523990] DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
[ 44.524826] DR6: fffe0ff0 DR7: 00000400
[ 44.525382] Call Trace:
[ 44.525768] ? alloc_indirect.isra.14+0x1b/0x40
[ 44.526402] alloc_indirect.isra.14+0x1b/0x40
[ 44.527025] virtqueue_add_sgs+0x208/0x460
[ 44.527610] virtio_queue_rq+0x163/0x310 [virtio_blk]
[ 44.528281] blk_mq_dispatch_rq_list+0x74/0x440
[ 44.528913] blk_mq_sched_dispatch_requests+0x184/0x190
[ 44.529639] __blk_mq_run_hw_queue+0x6f/0xb0
[ 44.530252] __blk_mq_delay_run_hw_queue+0x57/0x60
[ 44.530935] blk_mq_run_hw_queue+0x22/0x80
[ 44.531523] blk_mq_sched_insert_requests+0x73/0x80
[ 44.532207] blk_mq_flush_plug_list+0x1a8/0x220
[ 44.532899] ? __blk_mq_get_tag+0x23/0x90
[ 44.533465] blk_flush_plug_list+0xb9/0x1f0
[ 44.534063] blk_mq_make_request+0x3dc/0x570
[ 44.534700] generic_make_request+0xfc/0x2e0
[ 44.535344] submit_bio+0x67/0x130
[ 44.535868] ? __test_set_page_writeback+0x12c/0x2a0
[ 44.536538] ext4_io_submit+0x40/0x50
[ 44.537082] ext4_bio_write_page+0x208/0x4b0
[ 44.537713] mpage_submit_page+0x8c/0xc0
[ 44.538272] mpage_map_and_submit_extent+0x1fd/0x710
[ 44.538937] ext4_writepages+0x6d8/0x880
[ 44.539486] do_writepages+0x39/0xc0
[ 44.540000] ? rb_erase_cached+0x290/0x360
[ 44.540563] __filemap_fdatawrite_range+0xb4/0xe0
[ 44.541207] file_write_and_wait_range+0x55/0xa0
[ 44.541844] ext4_sync_file+0x101/0x3d0
[ 44.542389] ? ext4_getfsmap+0x330/0x330
[ 44.542941] vfs_fsync_range+0x3f/0xb0
[ 44.543490] do_fsync+0x2e/0x60
[ 44.543952] SyS_fsync+0x12/0x20
[ 44.544427] do_fast_syscall_32+0x7f/0x1e0
[ 44.545003] entry_SYSENTER_32+0x4e/0x7c
[ 44.545561] EIP: 0xb7ef2d09
[ 44.545988] EFLAGS: 00000282 CPU: 0
[ 44.546480] EAX: ffffffda EBX: 00000020 ECX: 00000002 EDX: 00000000
[ 44.547294] ESI: 00000000 EDI: 00000006 EBP: 00000002 ESP: b5d401a0
[ 44.548134] DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 007b
[ 44.548856] Code: 0f 84 b5 00 00 00 89 75 e4 8b 07 64 8b 50 04 64 03 05 28 91 df da 8b 08 85 c9 89 4d f0 0f 84 53 01 00 00 8b 4d f0 03 4f 14 8b 37 <8b> 01 33 87 b8 00 00 00 89 cb 89 4d e0 8d 4a 01 89 45 dc 31 c3
[ 44.551928] EIP: __kmalloc+0x85/0x220 SS:ESP: 0068:dcaf59d4
[ 44.552865] CR2: 00000000ee835a95
[ 44.553525] ---[ end trace 3c49b27dd79507a4 ]---

[Ubuntu Kernel Bot (ubuntu-kernel-bot)]

This bug is missing log files that will aid in diagnosing the problem. While running an Ubuntu kernel (not a mainline or third-party kernel) please enter the following command in a terminal window:

apport-collect 1814054

and then change the status of the bug to 'Confirmed'.

If, due to the nature of the issue you have encountered, you are unable to run this command, please add a comment stating that fact and change the bug status to 'Confirmed'.

This change has been made by an automated script, maintained by the Ubuntu Kernel Team.

[Juerg Haefliger (juergh)]

After doing some bisection I found the following commit to introduce the problem:

120645513f55a4ac5543120d9e79925d30a0156f is the first bad commit
commit 120645513f55a4ac5543120d9e79925d30a0156f
Author: Jarno Rajahalme <email address hidden>
Date: Fri Apr 21 16:48:06 2017 -0700

    openvswitch: Add eventmask support to CT action.

    Add a new optional conntrack action attribute OVS_CT_ATTR_EVENTMASK,
    which can be used in conjunction with the commit flag
    (OVS_CT_ATTR_COMMIT) to set the mask of bits specifying which
    conntrack events (IPCT_*) should be delivered via the Netfilter
    netlink multicast groups. Default behavior depends on the system
    configuration, but typically a lot of events are delivered. This can be
    very chatty for the NFNLGRP_CONNTRACK_UPDATE group, even if only some
    types of events are of interest.

    Netfilter core init_conntrack() adds the event cache extension, so we
    only need to set the ctmask value. However, if the system is
    configured without support for events, the setting will be skipped due
    to extension not being found.

    Signed-off-by: Jarno Rajahalme <email address hidden>
    Reviewed-by: Greg Rose <email address hidden>
    Acked-by: Joe Stringer <email address hidden>
    Signed-off-by: David S. Miller <email address hidden>

[Juerg Haefliger (juergh)]

I've also tested newer kernels and they're all susceptible to this (or a similar/related) problem. Sometimes I also see the following stack traces:

[ 125.300088] BUG: unable to handle kernel NULL pointer dereference at 00000074
[ 125.308050] IP: ma_put+0x25/0x40
[ 125.308551] *pdpt = 000000001b027001 *pde = 0000000000000000
[ 125.309388] Oops: 0002 [#1] SMP
[ 125.309885] Modules linked in: ufs msdos xfs openvswitch nsh nf_conntrack_ipv6 nf_nat_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_defrag_ipv6 nf_nat nf_conntrack isofs kvm_intel kvm joydev irqbypass input_leds serio_raw sch_fq_codel ib_iser rdma_cm iw_cm ib_cm ib_core iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi ip_tables x_tables autofs4 btrfs zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear psmouse virtio_blk virtio_net floppy
[ 125.316176] CPU: 0 PID: 1053 Comm: ovs-vswitchd Tainted: G W 4.15.0-43-generic #46-Ubuntu
[ 125.317585] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.11.1-1ubuntu1 04/01/2014
[ 125.318888] EIP: ma_put+0x25/0x40
[ 125.319428] EFLAGS: 00010246 CPU: 0
[ 125.319984] EAX: 00000000 EBX: db28ede0 ECX: 000000ff EDX: fffffe01
[ 125.320873] ESI: dc692c00 EDI: dc692c6c EBP: db01fb20 ESP: db01fb1c
[ 125.321765] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[ 125.322531] CR0: 80050033 CR2: 00000074 CR3: 1b9cfac0 CR4: 000006f0
[ 125.323471] DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
[ 125.324405] DR6: fffe0ff0 DR7: 00000400
[ 125.325018] Call Trace:
[ 125.325447] ipv6_mc_destroy_dev+0x4c/0x80
[ 125.326103] addrconf_ifdown+0x3c9/0x4c0
[ 125.326746] addrconf_notify+0x178/0x970
[ 125.327390] ? find_next_bit+0xa/0x10
[ 125.327999] ? cpumask_next+0x15/0x20
[ 125.328607] ? xfrm_policy_cache_flush+0x12f/0x14a
[ 125.329364] ? fib_add_ifaddr+0x170/0x170
[ 125.330031] ? inet6_ifinfo_notify+0xb0/0xb0
[ 125.330718] notifier_call_chain+0x51/0x80
[ 125.331380] raw_notifier_call_chain+0x11/0x20
[ 125.332092] call_netdevice_notifiers_info+0x25/0x50
[ 125.332954] rollback_registered_many+0x21f/0x390
[ 125.333725] unregister_netdevice_queue+0x74/0xe0
[ 125.334451] internal_dev_destroy+0x32/0x50 [openvswitch]
[ 125.335210] ovs_vport_del+0x39/0x40 [openvswitch]
[ 125.335917] __dp_destroy+0x90/0xc0 [openvswitch]
[ 125.336649] ovs_dp_cmd_del+0x71/0xd0 [openvswitch]
[ 125.337406] genl_rcv_msg+0x1fe/0x3a0
[ 125.338074] ? update_curr+0x80/0x240
[ 125.338715] ? genl_rcv+0x30/0x30
[ 125.339257] netlink_rcv_skb+0x6e/0xf0
[ 125.339860] genl_rcv+0x21/0x30
[ 125.340377] netlink_unicast+0x16f/0x200
[ 125.341001] netlink_sendmsg+0x247/0x390
[ 125.341633] ? netlink_unicast+0x200/0x200
[ 125.342309] sock_sendmsg+0x32/0x40
[ 125.342892] ___sys_sendmsg+0x249/0x260
[ 125.343522] ? default_wake_function+0x10/0x20
[ 125.344236] ? pollwake+0x68/0x90
[ 125.344796] ? wake_up_q+0x60/0x60
[ 125.345368] ? current_time+0x39/0x70
[ 125.345983] ? __wake_up_common_lock+0x82/0xb0
[ 125.346697] ? __atime_needs_update+0x7a/0x160
[ 125.347407] ? touch_atime+0x2b/0xb0
[ 125.348002] ? __wake_up_...