Hi Andrew,
I just retest this manually on node entei with the steps in comment #3, and this issue can be reproduced (system gets reboot) with a different message from the ipmi console.
[ 417.696448] BUG: Unable to handle kernel instruction fetch (NULL pointer?) [ 417.696522] Faulting instruction address: 0x00000000 [ 417.696677] Oops: Kernel access of bad area, sig: 11 [#1] [ 417.696693] LE PAGE_SIZE=64K MMU=Hash SMP NR_CPUS=2048 NUMA PowerNV [ 417.696715] Modules linked in: binfmt_misc dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_alua joydev input_leds mac_hid ofpart plx_dma cmdlinepart ipmi_powernv pow ernv_flash ipmi_devintf ibmpowernv at24 vmx_crypto opal_prd ipmi_msghandler powernv_rng mtd uio_pdrv_genirq uio sch_fq_codel ip_tables x_tables autofs4 btrfs blake2b_ generic uas raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor hid_generic usbhid hid usb_storage async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear ast drm_vram_helper i2c_algo_bit drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_vpmsum cec crc32c_vpmsum rc_core drm ahci tg3 xhci_pci libahci drm_panel_orientation_quirks xhci_pci_renesas [ 417.697008] CPU: 0 PID: 3117 Comm: reuseport_bpf_c Not tainted 5.11.0-27-generic #29~20.04.1-Ubuntu [ 417.697034] NIP: 0000000000000000 LR: c000000000e77ba8 CTR: 0000000000000000 [ 417.697055] REGS: c0000007ff6e74d0 TRAP: 0400 Not tainted (5.11.0-27-generic) [ 417.697077] MSR: 9000000040009033 <SF,HV,EE,ME,IR,DR,RI,LE> CR: 28022444 XER: 20000000 [ 417.697309] CFAR: c000000000010300 IRQMASK: 0 [ 417.697309] GPR00: c000000000e77b80 c0000007ff6e7770 c000000001e99600 c000000014cecd00 [ 417.697309] GPR04: c008000004230038 c000000014cecd00 0000000000000008 0000000000000001 [ 417.697309] GPR08: 0000000000000001 0000000000000000 c0000000501e9580 0000000000000000 [ 417.697309] GPR12: 0000000000000000 c000000002150000 0000000000000000 0000000000000000 [ 417.697309] GPR16: 0000000000000040 c00000078e09a480 0000000000000001 0000000000000001 [ 417.697309] GPR20: 00000000000022b8 0000000000000000 000000000000cfb3 000000000100007f [ 417.697309] GPR24: 0000000000000000 0000000000000008 c000000001dba880 c008000004230000 [ 417.697309] GPR28: 0000000000000080 c000000003292000 0000000090dd40dc c000000014cecd00 [ 417.697503] NIP [0000000000000000] 0x0 [ 417.697517] LR [c000000000e77ba8] reuseport_select_sock+0x108/0x3f0 [ 417.697541] Call Trace: [ 417.697550] [c0000007ff6e7810] [c000000000f64314] udp4_lib_lookup2+0x1a4/0x2b0 [ 417.697576] [c0000007ff6e7890] [c000000000f65928] __udp4_lib_lookup+0x358/0x540 [ 417.697602] [c0000007ff6e79d0] [c000000000f66978] __udp4_lib_rcv+0x608/0xe10 [ 417.697626] [c0000007ff6e7af0] [c000000000f0fa20] ip_protocol_deliver_rcu+0x60/0x2c0 [ 417.697813] [c0000007ff6e7b40] [c000000000f0fcf0] ip_local_deliver_finish+0x70/0x90 [ 417.697838] [c0000007ff6e7b60] [c000000000f0fda0] ip_local_deliver+0x90/0x180 [ 417.697861] [c0000007ff6e7be0] [c000000000f0f140] ip_rcv_finish+0xc0/0xf0 [ 417.697883] [c0000007ff6e7c20] [c000000000f0ffa8] ip_rcv+0x118/0x130 [ 417.697904] [c0000007ff6e7ca0] [c000000000e3a3b4] __netif_receive_skb_one_core+0x74/0xb0 [ 417.698083] [c0000007ff6e7cf0] [c000000000e3a770] process_backlog+0xd0/0x230 [ 417.698108] [c0000007ff6e7d60] [c000000000e3cf88] net_rx_action+0x1e8/0x580 [ 417.698130] [c0000007ff6e7e70] [c0000000010a94c0] __do_softirq+0x160/0x404 [ 417.698152] [c0000007ff6e7f90] [c00000000002bad8] call_do_softirq+0x14/0x24 [ 417.698174] [c00000005021b760] [c000000000017158] do_softirq_own_stack+0x38/0x50 [ 417.698199] [c00000005021b780] [c0000000001576e0] do_softirq+0xa0/0xb0 [ 417.698391] [c00000005021b7b0] [c0000000001577e8] __local_bh_enable_ip+0xf8/0x120 [ 417.698415] [c00000005021b7d0] [c000000000f1418c] ip_finish_output2+0x1fc/0x730 [ 417.698439] [c00000005021b870] [c000000000f178fc] ip_output+0xdc/0x1c0 [ 417.698460] [c00000005021b920] [c000000000f16cb4] ip_local_out+0x64/0x90 [ 417.698482] [c00000005021b960] [c000000000f18634] ip_send_skb+0x34/0xc0 [ 417.698667] [c00000005021b990] [c000000000f6074c] udp_send_skb.isra.0+0x16c/0x4a0 [ 417.698691] [c00000005021b9e0] [c000000000f61490] udp_sendmsg+0x960/0xcf0 [ 417.698714] [c00000005021bbd0] [c000000000f77cd4] inet_sendmsg+0x64/0xb0 [ 417.698735] [c00000005021bc10] [c000000000dfb3a0] sock_sendmsg+0x80/0xb0 [ 417.698758] [c00000005021bc40] [c000000000dfffd8] __sys_sendto+0xf8/0x1b0 [ 417.698910] [c00000005021bd90] [c000000000e00100] sys_send+0x30/0x40 [ 417.698932] [c00000005021bdb0] [c000000000036204] system_call_exception+0xf4/0x200 [ 417.698999] [c00000005021be10] [c00000000000d860] system_call_common+0xf0/0x27c [ 417.699023] --- interrupt: c00 at 0x7aef2b250724 [ 417.699040] NIP: 00007aef2b250724 LR: 000005f77e131568 CTR: 0000000000000000 [ 417.716065] REGS: c00000005021be80 TRAP: 0c00 Not tainted (5.11.0-27-generic) [ 417.716693] MSR: 900000000280f033 <SF,HV,VEC,VSX,EE,PR,FP,ME,IR,DR,RI,LE> CR: 28004440 XER: 00000000 [ 417.717786] IRQMASK: 0 [ 417.717786] GPR00: 000000000000014e 00007fffed1b68e0 00007aef2b337100 0000000000000084 [ 417.717786] GPR04: 000005f77e131f08 0000000000000001 0000000000000000 0000000000000000 [ 417.717786] GPR08: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 [ 417.717786] GPR12: 0000000000000000 00007aef2b3ca330 0000000000000000 0000000000000000 [ 417.717786] GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 [ 417.717786] GPR20: 0000000000000001 000005f7b987049c 000000000000007f 00007fffed1b6b88 [ 417.717786] GPR24: 000005f7b987029c 0000000000000083 000005f7b98702a0 0000000000000002 [ 417.717786] GPR28: 0000000000000000 0000000000000080 0000000000000002 0000000000000084 [ 417.735682] NIP [00007aef2b250724] 0x7aef2b250724 [ 417.736172] LR [000005f77e131568] 0x5f77e131568 [ 417.736699] --- interrupt: c00 [ 417.736710] Instruction dump: [ 417.737229] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [ 417.737755] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [ 417.738770] ---[ end trace 9fc532dd6c3c783a ]--- [ 417.900679] [ 418.900837] Kernel panic - not syncing: Aiee, killing interrupt handler! [ 419.063147] Rebooting in 10 seconds.. [ 481.430090023,5] OPAL: Reboot request...
Hi Andrew,
I just retest this manually on node entei with the steps in comment #3, and this issue can be reproduced (system gets reboot) with a different message from the ipmi console.
[ 417.696448] BUG: Unable to handle kernel instruction fetch (NULL pointer?) orientation_ quirks xhci_pci_renesas EE,ME,IR, DR,RI,LE> CR: 28022444 XER: 20000000 select_ sock+0x108/ 0x3f0 lookup2+ 0x1a4/0x2b0 lib_lookup+ 0x358/0x540 lib_rcv+ 0x608/0xe10 deliver_ rcu+0x60/ 0x2c0 deliver_ finish+ 0x70/0x90 deliver+ 0x90/0x180 finish+ 0xc0/0xf0 receive_ skb_one_ core+0x74/ 0xb0 backlog+ 0xd0/0x230 action+ 0x1e8/0x580 0x160/0x404 softirq+ 0x14/0x24 own_stack+ 0x38/0x50 0xa0/0xb0 bh_enable_ ip+0xf8/ 0x120 output2+ 0x1fc/0x730 0xdc/0x1c0 out+0x64/ 0x90 skb+0x34/ 0xc0 skb.isra. 0+0x16c/ 0x4a0 0x960/0xcf0 0x64/0xb0 0x80/0xb0 0xf8/0x1b0 call_exception+ 0xf4/0x200 call_common+ 0xf0/0x27c VEC,VSX, EE,PR,FP, ME,IR,DR, RI,LE> CR: 28004440 XER: 00000000
[ 417.696522] Faulting instruction address: 0x00000000
[ 417.696677] Oops: Kernel access of bad area, sig: 11 [#1]
[ 417.696693] LE PAGE_SIZE=64K MMU=Hash SMP NR_CPUS=2048 NUMA PowerNV
[ 417.696715] Modules linked in: binfmt_misc dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_alua joydev input_leds mac_hid ofpart plx_dma cmdlinepart ipmi_powernv pow
ernv_flash ipmi_devintf ibmpowernv at24 vmx_crypto opal_prd ipmi_msghandler powernv_rng mtd uio_pdrv_genirq uio sch_fq_codel ip_tables x_tables autofs4 btrfs blake2b_
generic uas raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor hid_generic usbhid hid usb_storage async_tx xor raid6_pq libcrc32c raid1 raid0 multipath
linear ast drm_vram_helper i2c_algo_bit drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_vpmsum cec crc32c_vpmsum rc_core drm
ahci tg3 xhci_pci libahci drm_panel_
[ 417.697008] CPU: 0 PID: 3117 Comm: reuseport_bpf_c Not tainted 5.11.0-27-generic #29~20.04.1-Ubuntu
[ 417.697034] NIP: 0000000000000000 LR: c000000000e77ba8 CTR: 0000000000000000
[ 417.697055] REGS: c0000007ff6e74d0 TRAP: 0400 Not tainted (5.11.0-27-generic)
[ 417.697077] MSR: 9000000040009033 <SF,HV,
[ 417.697309] CFAR: c000000000010300 IRQMASK: 0
[ 417.697309] GPR00: c000000000e77b80 c0000007ff6e7770 c000000001e99600 c000000014cecd00
[ 417.697309] GPR04: c008000004230038 c000000014cecd00 0000000000000008 0000000000000001
[ 417.697309] GPR08: 0000000000000001 0000000000000000 c0000000501e9580 0000000000000000
[ 417.697309] GPR12: 0000000000000000 c000000002150000 0000000000000000 0000000000000000
[ 417.697309] GPR16: 0000000000000040 c00000078e09a480 0000000000000001 0000000000000001
[ 417.697309] GPR20: 00000000000022b8 0000000000000000 000000000000cfb3 000000000100007f
[ 417.697309] GPR24: 0000000000000000 0000000000000008 c000000001dba880 c008000004230000
[ 417.697309] GPR28: 0000000000000080 c000000003292000 0000000090dd40dc c000000014cecd00
[ 417.697503] NIP [0000000000000000] 0x0
[ 417.697517] LR [c000000000e77ba8] reuseport_
[ 417.697541] Call Trace:
[ 417.697550] [c0000007ff6e7810] [c000000000f64314] udp4_lib_
[ 417.697576] [c0000007ff6e7890] [c000000000f65928] __udp4_
[ 417.697602] [c0000007ff6e79d0] [c000000000f66978] __udp4_
[ 417.697626] [c0000007ff6e7af0] [c000000000f0fa20] ip_protocol_
[ 417.697813] [c0000007ff6e7b40] [c000000000f0fcf0] ip_local_
[ 417.697838] [c0000007ff6e7b60] [c000000000f0fda0] ip_local_
[ 417.697861] [c0000007ff6e7be0] [c000000000f0f140] ip_rcv_
[ 417.697883] [c0000007ff6e7c20] [c000000000f0ffa8] ip_rcv+0x118/0x130
[ 417.697904] [c0000007ff6e7ca0] [c000000000e3a3b4] __netif_
[ 417.698083] [c0000007ff6e7cf0] [c000000000e3a770] process_
[ 417.698108] [c0000007ff6e7d60] [c000000000e3cf88] net_rx_
[ 417.698130] [c0000007ff6e7e70] [c0000000010a94c0] __do_softirq+
[ 417.698152] [c0000007ff6e7f90] [c00000000002bad8] call_do_
[ 417.698174] [c00000005021b760] [c000000000017158] do_softirq_
[ 417.698199] [c00000005021b780] [c0000000001576e0] do_softirq+
[ 417.698391] [c00000005021b7b0] [c0000000001577e8] __local_
[ 417.698415] [c00000005021b7d0] [c000000000f1418c] ip_finish_
[ 417.698439] [c00000005021b870] [c000000000f178fc] ip_output+
[ 417.698460] [c00000005021b920] [c000000000f16cb4] ip_local_
[ 417.698482] [c00000005021b960] [c000000000f18634] ip_send_
[ 417.698667] [c00000005021b990] [c000000000f6074c] udp_send_
[ 417.698691] [c00000005021b9e0] [c000000000f61490] udp_sendmsg+
[ 417.698714] [c00000005021bbd0] [c000000000f77cd4] inet_sendmsg+
[ 417.698735] [c00000005021bc10] [c000000000dfb3a0] sock_sendmsg+
[ 417.698758] [c00000005021bc40] [c000000000dfffd8] __sys_sendto+
[ 417.698910] [c00000005021bd90] [c000000000e00100] sys_send+0x30/0x40
[ 417.698932] [c00000005021bdb0] [c000000000036204] system_
[ 417.698999] [c00000005021be10] [c00000000000d860] system_
[ 417.699023] --- interrupt: c00 at 0x7aef2b250724
[ 417.699040] NIP: 00007aef2b250724 LR: 000005f77e131568 CTR: 0000000000000000
[ 417.716065] REGS: c00000005021be80 TRAP: 0c00 Not tainted (5.11.0-27-generic)
[ 417.716693] MSR: 900000000280f033 <SF,HV,
[ 417.717786] IRQMASK: 0
[ 417.717786] GPR00: 000000000000014e 00007fffed1b68e0 00007aef2b337100 0000000000000084
[ 417.717786] GPR04: 000005f77e131f08 0000000000000001 0000000000000000 0000000000000000
[ 417.717786] GPR08: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 417.717786] GPR12: 0000000000000000 00007aef2b3ca330 0000000000000000 0000000000000000
[ 417.717786] GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 417.717786] GPR20: 0000000000000001 000005f7b987049c 000000000000007f 00007fffed1b6b88
[ 417.717786] GPR24: 000005f7b987029c 0000000000000083 000005f7b98702a0 0000000000000002
[ 417.717786] GPR28: 0000000000000000 0000000000000080 0000000000000002 0000000000000084
[ 417.735682] NIP [00007aef2b250724] 0x7aef2b250724
[ 417.736172] LR [000005f77e131568] 0x5f77e131568
[ 417.736699] --- interrupt: c00
[ 417.736710] Instruction dump:
[ 417.737229] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
[ 417.737755] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
[ 417.738770] ---[ end trace 9fc532dd6c3c783a ]---
[ 417.900679]
[ 418.900837] Kernel panic - not syncing: Aiee, killing interrupt handler!
[ 419.063147] Rebooting in 10 seconds..
[ 481.430090023,5] OPAL: Reboot request...