Still seeing guestOS panics during detach below: https://53a68ba6d05d22d0f44d-c5982ef1d1780edfccf5471d12724c1b.ssl.cf5.rackcdn.com/812473/2/check/nova-grenade-multinode/31101fd/testr_results.html 2021-10-07 12:36:45,955 223284 INFO [tempest.lib.common.rest_client] Request (LiveMigrationTest:_run_cleanups): 200 POST http://10.210.192.135/compute/v2.1/servers/503a61f0-8900-4f2b-b7c4-5bd6402f575b/action 0.051s 2021-10-07 12:36:45,956 223284 DEBUG [tempest.lib.common.rest_client] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: {"os-getConsoleOutput": {}} Response - Headers: {'date': 'Thu, 07 Oct 2021 12:36:45 GMT', 'server': 'Apache/2.4.41 (Ubuntu)', 'content-length': '9549', 'content-type': 'application/json', 'openstack-api-version': 'compute 2.1', 'x-openstack-nova-api-version': '2.1', 'vary': 'OpenStack-API-Version,X-OpenStack-Nova-API-Version', 'x-openstack-request-id': 'req-818114fc-1e1e-4e46-9748-de28184d4780', 'x-compute-request-id': 'req-818114fc-1e1e-4e46-9748-de28184d4780', 'connection': 'close', 'status': '200', 'content-location': 'http://10.210.192.135/compute/v2.1/servers/503a61f0-8900-4f2b-b7c4-5bd6402f575b/action'} Body: b'{"output": "[ 16.588694] BUG: unable to handle page fault for address: 0000000000461c36\\n[ 16.593560] #PF: supervisor read access in user mode\\n[ 16.593868] #PF: error_code(0x6e7860) - not-present page\\n[ 16.594289] IDT: 0xfffffe0000000000 (limit=0xfff) GDT: 0xfffffe0000001000 (limit=0x7f)\\n[ 16.594773] LDTR: NULL\\n[ 16.595779] TR: 0x40 -- base=0xfffffe0000003000 limit=0x206f\\n[ 16.596191] PGD 2da7067 P4D 2da7067 PUD 2da6067 PMD 3ce6067 PTE 0\\n[ 16.597308] Oops: 7860 [#1] SMP NOPTI\\n[ 16.597949] CPU: 0 PID: 289 Comm: wait_iface Not tainted 5.3.0-26-generic #28~18.04.1-Ubuntu\\n[ 16.598345] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.13.0-1ubuntu1.1 04/01/2014\\n[ 16.599745] RIP: 0033:0x435117\\n[ 16.600260] Code: ff ff 48 89 df e8 c9 23 fd ff 5b e9 da fb ff ff 48 8b 15 0c 7e 27 00 53 8b 42 38 ff c0 89 42 38 bf 10 00 00 00 e8 b0 3a fd ff <48> 89 c3 48 8b 05 bf 7c 27 00 48 89 1d b8 7c 27 00 48 89 03 e8 a7\\n[ 16.601054] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246\\n[ 16.601368] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010\\n[ 16.601710] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007\\n[ 16.602049] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a\\n[ 16.602380] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0\\n[ 16.602717] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000\\n[ 16.603127] FS: 00007fce1b5506a0 GS: 0000000000000000\\n[ 16.603445] Modules linked in: ahci libahci ip_tables x_tables nls_utf8 nls_iso8859_1 nls_ascii isofs hid_generic usbhid hid virtio_rng virtio_gpu drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm virtio_scsi virtio_net net_failover failover virtio_input virtio_blk qemu_fw_cfg 9pnet_virtio 9pnet pcnet32 8139cp mii ne2k_pci 8390 e1000\\n[ 16.605249] CR2: 0000000000461c36\\n[ 16.606644] ---[ end trace fafa623ec1329fa4 ]---\\n[ 16.607001] RIP: 0033:0x435117\\n[ 16.607209] Code: ff ff 48 89 df e8 c9 23 fd ff 5b e9 da fb ff ff 48 8b 15 0c 7e 27 00 53 8b 42 38 ff c0 89 42 38 bf 10 00 00 00 e8 b0 3a fd ff <48> 89 c3 48 8b 05 bf 7c 27 00 48 89 1d b8 7c 27 00 48 89 03 e8 a7\\n[ 16.607943] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246\\n[ 16.608192] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010\\n[ 16.608503] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007\\n[ 16.608811] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a\\n[ 16.609118] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0\\n[ 16.609428] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000\\n[ 16.609828] FS: 00007fce1b5506a0(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000\\n[ 16.610200] CS: 0033 DS: 0000 ES: 0000 CR0: 0000000080050033\\n[ 16.610921] CR2: 0000000000461c36 CR3: 0000000003d26000 CR4: 00000000000006f0\\n[ 16.639815] ------------[ cut here ]------------\\n[ 16.640449] kernel BUG at /build/linux-hwe-3HpQOB/linux-hwe-5.3.0/include/linux/fs.h:2902!\\n[ 16.641191] invalid opcode: 0000 [#2] SMP NOPTI\\n[ 16.641525] CPU: 0 PID: 289 Comm: wait_iface Tainted: G D 5.3.0-26-generic #28~18.04.1-Ubuntu\\n[ 16.642179] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.13.0-1ubuntu1.1 04/01/2014\\n[ 16.643303] RIP: 0010:__fput+0x252/0x260\\n[ 16.643613] Code: 0f 84 ab fe ff ff 41 f7 c4 00 40 00 00 0f 85 9e fe ff ff e8 60 38 00 00 e9 94 fe ff ff 4c 89 f7 e8 83 5a 02 00 e9 d2 fe ff ff <0f> 0b 66 90 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 31 ff\\n[ 16.644827] RSP: 0018:ffffb1b5c0273e40 EFLAGS: 00000246\\n[ 16.645112] RAX: 0000000000000000 RBX: ffff8e5083d46200 RCX: 0000000000000001\\n[ 16.645431] RDX: ffff8e5083d462d0 RSI: ffff8e5083d46200 RDI: 0000000000000000\\n[ 16.645747] RBP: ffffb1b5c0273e80 R08: 0000000000000000 R09: 0000000000000000\\n[ 16.646069] R10: 0000000000000010 R11: ffff8e508418f7c0 R12: 00000000080a801d\\n[ 16.646385] R13: ffff8e508418f7c0 R14: ffff8e5082d813e0 R15: ffff8e5084056240\\n[ 16 2021-10-07 12:36:45,957 223284 DEBUG [tempest.common.waiters] Console output for 503a61f0-8900-4f2b-b7c4-5bd6402f575b body= [ 16.588694] BUG: unable to handle page fault for address: 0000000000461c36 [ 16.593560] #PF: supervisor read access in user mode [ 16.593868] #PF: error_code(0x6e7860) - not-present page [ 16.594289] IDT: 0xfffffe0000000000 (limit=0xfff) GDT: 0xfffffe0000001000 (limit=0x7f) [ 16.594773] LDTR: NULL [ 16.595779] TR: 0x40 -- base=0xfffffe0000003000 limit=0x206f [ 16.596191] PGD 2da7067 P4D 2da7067 PUD 2da6067 PMD 3ce6067 PTE 0 [ 16.597308] Oops: 7860 [#1] SMP NOPTI [ 16.597949] CPU: 0 PID: 289 Comm: wait_iface Not tainted 5.3.0-26-generic #28~18.04.1-Ubuntu [ 16.598345] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.13.0-1ubuntu1.1 04/01/2014 [ 16.599745] RIP: 0033:0x435117 [ 16.600260] Code: ff ff 48 89 df e8 c9 23 fd ff 5b e9 da fb ff ff 48 8b 15 0c 7e 27 00 53 8b 42 38 ff c0 89 42 38 bf 10 00 00 00 e8 b0 3a fd ff <48> 89 c3 48 8b 05 bf 7c 27 00 48 89 1d b8 7c 27 00 48 89 03 e8 a7 [ 16.601054] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246 [ 16.601368] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010 [ 16.601710] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007 [ 16.602049] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a [ 16.602380] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0 [ 16.602717] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000 [ 16.603127] FS: 00007fce1b5506a0 GS: 0000000000000000 [ 16.603445] Modules linked in: ahci libahci ip_tables x_tables nls_utf8 nls_iso8859_1 nls_ascii isofs hid_generic usbhid hid virtio_rng virtio_gpu drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm virtio_scsi virtio_net net_failover failover virtio_input virtio_blk qemu_fw_cfg 9pnet_virtio 9pnet pcnet32 8139cp mii ne2k_pci 8390 e1000 [ 16.605249] CR2: 0000000000461c36 [ 16.606644] ---[ end trace fafa623ec1329fa4 ]--- [ 16.607001] RIP: 0033:0x435117 [ 16.607209] Code: ff ff 48 89 df e8 c9 23 fd ff 5b e9 da fb ff ff 48 8b 15 0c 7e 27 00 53 8b 42 38 ff c0 89 42 38 bf 10 00 00 00 e8 b0 3a fd ff <48> 89 c3 48 8b 05 bf 7c 27 00 48 89 1d b8 7c 27 00 48 89 03 e8 a7 [ 16.607943] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246 [ 16.608192] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010 [ 16.608503] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007 [ 16.608811] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a [ 16.609118] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0 [ 16.609428] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000 [ 16.609828] FS: 00007fce1b5506a0(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000 [ 16.610200] CS: 0033 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 16.610921] CR2: 0000000000461c36 CR3: 0000000003d26000 CR4: 00000000000006f0 [ 16.639815] ------------[ cut here ]------------ [ 16.640449] kernel BUG at /build/linux-hwe-3HpQOB/linux-hwe-5.3.0/include/linux/fs.h:2902! [ 16.641191] invalid opcode: 0000 [#2] SMP NOPTI [ 16.641525] CPU: 0 PID: 289 Comm: wait_iface Tainted: G D 5.3.0-26-generic #28~18.04.1-Ubuntu [ 16.642179] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.13.0-1ubuntu1.1 04/01/2014 [ 16.643303] RIP: 0010:__fput+0x252/0x260 [ 16.643613] Code: 0f 84 ab fe ff ff 41 f7 c4 00 40 00 00 0f 85 9e fe ff ff e8 60 38 00 00 e9 94 fe ff ff 4c 89 f7 e8 83 5a 02 00 e9 d2 fe ff ff <0f> 0b 66 90 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 31 ff [ 16.644827] RSP: 0018:ffffb1b5c0273e40 EFLAGS: 00000246 [ 16.645112] RAX: 0000000000000000 RBX: ffff8e5083d46200 RCX: 0000000000000001 [ 16.645431] RDX: ffff8e5083d462d0 RSI: ffff8e5083d46200 RDI: 0000000000000000 [ 16.645747] RBP: ffffb1b5c0273e80 R08: 0000000000000000 R09: 0000000000000000 [ 16.646069] R10: 0000000000000010 R11: ffff8e508418f7c0 R12: 00000000080a801d [ 16.646385] R13: ffff8e508418f7c0 R14: ffff8e5082d813e0 R15: ffff8e5084056240 [ 16.646710] FS: 0000000000000000(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000 [ 16.647126] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 16.647542] CR2: 00007fce1b550980 CR3: 0000000005a0a000 CR4: 00000000000006f0 [ 16.648126] Call Trace: [ 16.649690] ____fput+0xe/0x10 [ 16.650219] task_work_run+0x9d/0xc0 [ 16.650439] do_exit+0x2f4/0xba0 [ 16.650648] rewind_stack_do_exit+0x17/0x20 [ 16.650994] RIP: 0033:0x435117 [ 16.651604] Code: Bad RIP value. [ 16.651838] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246 [ 16.652358] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010 [ 16.652881] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007 [ 16.653281] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a [ 16.653608] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0 [ 16.653930] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000 [ 16.654458] Modules linked in: ahci libahci ip_tables x_tables nls_utf8 nls_iso8859_1 nls_ascii isofs hid_generic usbhid hid virtio_rng virtio_gpu drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm virtio_scsi virtio_net net_failover failover virtio_input virtio_blk qemu_fw_cfg 9pnet_virtio 9pnet pcnet32 8139cp mii ne2k_pci 8390 e1000 [ 16.656856] ---[ end trace fafa623ec1329fa5 ]--- [ 16.657202] RIP: 0033:0x435117 [ 16.657557] Code: Bad RIP value. [ 16.657980] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246 [ 16.658321] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010 [ 16.659023] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007 [ 16.659439] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a [ 16.659801] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0 [ 16.660299] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000 [ 16.660828] FS: 0000000000000000(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000 [ 16.661202] CS: 0033 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 16.661469] CR2: 00000000004350ed CR3: 0000000005a0a000 CR4: 00000000000006f0 [ 16.661945] Fixing recursive fault but reboot is needed! [ 22.072466] general protection fault: 0000 [#3] SMP NOPTI [ 22.072841] CPU: 0 PID: 13 Comm: kworker/0:1 Tainted: G D 5.3.0-26-generic #28~18.04.1-Ubuntu [ 22.073289] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.13.0-1ubuntu1.1 04/01/2014 [ 22.073895] Workqueue: kacpid acpi_os_execute_deferred [ 22.074250] RIP: 0010:__kmalloc+0xa5/0x260 [ 22.074662] Code: 65 49 8b 50 08 65 4c 03 05 b0 7a 37 71 4d 8b 38 4d 85 ff 0f 84 82 01 00 00 41 8b 59 20 49 8b 39 48 8d 4a 01 4c 89 f8 4c 01 fb <48> 33 1b 49 33 99 70 01 00 00 65 48 0f c7 0f 0f 94 c0 84 c0 74 bd [ 22.075479] RSP: 0018:ffffb1b5c0073d28 EFLAGS: 00000206 [ 22.075728] RAX: 6ced839a910c620c RBX: 6ced839a910c620c RCX: 000000000000106a [ 22.076041] RDX: 0000000000001069 RSI: 0000000000000dc0 RDI: 000000000002f060 [ 22.076358] RBP: ffffb1b5c0073d58 R08: ffff8e508762f060 R09: ffff8e5087001a80 [ 22.076668] R10: ffff8e5082aa3d60 R11: fefefefefefefeff R12: 0000000000000dc0 [ 22.076979] R13: 000000000000000b R14: ffff8e5087001a80 R15: 6ced839a910c620c [ 22.077301] FS: 0000000000000000(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000 [ 22.077650] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 22.077910] CR2: 00000000004350ed CR3: 0000000005a0a000 CR4: 00000000000006f0 [ 22.078228] Call Trace: [ 22.078434] ? acpi_os_allocate_zeroed+0x2d/0x2f [ 22.078703] acpi_os_allocate_zeroed+0x2d/0x2f [ 22.078951] acpi_ns_get_normalized_pathname+0x7f/0x140 [ 22.079241] acpi_ex_start_trace_method+0x20/0xb9 [ 22.079484] acpi_ds_begin_method_execution+0x7c/0x387 [ 22.079764] acpi_ps_execute_method+0x7c/0x2d1 [ 22.079998] acpi_ns_evaluate+0x34b/0x4ed [ 22.080213] acpi_ev_asynch_execute_gpe_method+0xbd/0x159 [ 22.080480] acpi_os_execute_deferred+0x1a/0x30 [ 22.080715] process_one_work+0x1fd/0x3f0 [ 22.080933] worker_thread+0x34/0x410 [ 22.081137] kthread+0x121/0x140 [ 22.081325] ? process_one_work+0x3f0/0x3f0 [ 22.081541] ? kthread_park+0xb0/0xb0 [ 22.081747] ret_from_fork+0x35/0x40 [ 22.081990] Modules linked in: ahci libahci ip_tables x_tables nls_utf8 nls_iso8859_1 nls_ascii isofs hid_generic usbhid hid virtio_rng virtio_gpu drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm virtio_scsi virtio_net net_failover failover virtio_input virtio_blk qemu_fw_cfg 9pnet_virtio 9pnet pcnet32 8139cp mii ne2k_pci 8390 e1000 [ 22.083510] ---[ end trace fafa623ec1329fa6 ]--- [ 22.083760] RIP: 0033:0x435117 [ 22.083974] Code: Bad RIP value. [ 22.084147] RSP: 002b:00007ffce3276f50 EFLAGS: 00000246 [ 22.084397] RAX: 00000000004822a0 RBX: 00000000006e4010 RCX: 00000000006e4010 [ 22.084710] RDX: 00007ffce3276eb0 RSI: 00000000006e77d0 RDI: 0000000000000007 [ 22.085022] RBP: 0000000000000000 R08: fefefefefefefeff R09: fefefefefefeff5a [ 22.085328] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000006e44b0 [ 22.085636] R13: 0000000000000000 R14: 00007ffce3276fa8 R15: 0000000000000000 [ 22.085948] FS: 0000000000000000(0000) GS:ffff8e5087600000(0000) knlGS:0000000000000000 [ 22.086295] CS: 0033 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 22.086704] CR2: 00000000004350ed CR3: 0000000005a0a000 CR4: 00000000000006f0