Repeating kernel BUG at mm/migrate.c:654 after upgrade to 6.5

[Ariel E]

After the upgrade to 22.04.4, and kernel linux-signed-aws-6.5, we are getting the following bug in dmesg:

[176614.768600] kernel BUG at mm/migrate.c:654!
[176614.770487] invalid opcode: 0000 [#1] SMP NOPTI
[176614.772485] CPU: 252 PID: 1306651 Comm: memclean Tainted: P O 6.5.0-1011-aws #11~22.04.1-Ubuntu
[176614.776434] Hardware name: Amazon EC2 u-12tb1.112xlarge/, BIOS 1.0 10/16/2017
[176614.779305] RIP: 0010:migrate_folio_extra+0x85/0x90
[176614.781277] Code: 31 ff 45 31 c0 c3 cc cc cc cc e8 06 d2 ff ff 44 89 f0 5b 41 5c 41 5d 41 5e 5d 31 d2 31 c9 31 f6 31 ff 45 31 c0 c3 cc cc cc cc <0f> 0b 66 0f 1f 84 00 00 00 00 00 90 90 90 90 90 90 90 90 90 90 90
[176614.788556] RSP: 0018:ffffb0c7f366b918 EFLAGS: 00010282
[176614.790673] RAX: 0157ffffc4008067 RBX: ffffdcdf963e9340 RCX: 0000000000000002
[176614.793567] RDX: ffffdcdf963e9340 RSI: ffffdce09cb61240 RDI: ffff98e202addcd0
[176614.796485] RBP: ffffb0c7f366b940 R08: 0000000000000000 R09: 0000000000000000
[176614.799366] R10: 0000000000000000 R11: 0000000000000000 R12: ffff98e202addcd0
[176614.802124] R13: ffffdce09cb61240 R14: 0000000000000002 R15: ffffb0c7f366b9c0
[176614.805121] FS: 00007f936c48c740(0000) GS:ffff943a72b00000(0000) knlGS:0000000000000000
[176614.811858] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[176614.815921] CR2: 00005574a10bb790 CR3: 000004976529a004 CR4: 00000000007706e0
[176614.822250] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[176614.828746] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[176614.835165] PKRU: 55555554
[176614.838103] Call Trace:
[176614.841080] <TASK>
[176614.843889] ? show_regs+0x72/0x90
[176614.847138] ? die+0x38/0xb0
[176614.850149] ? do_trap+0xe3/0x100
[176614.853460] ? do_error_trap+0x75/0xb0
[176614.856909] ? migrate_folio_extra+0x85/0x90
[176614.860470] ? exc_invalid_op+0x53/0x80
[176614.863884] ? migrate_folio_extra+0x85/0x90
[176614.867419] ? asm_exc_invalid_op+0x1b/0x20
[176614.870925] ? migrate_folio_extra+0x85/0x90
[176614.874525] ? move_to_new_folio+0x145/0x160
[176614.878109] migrate_pages_batch+0x610/0x930
[176614.881776] ? __pfx_compaction_free+0x10/0x10
[176614.885466] ? __pfx_remove_migration_pte+0x10/0x10
[176614.889267] migrate_pages_sync+0x14e/0x200
[176614.892849] ? __pfx_compaction_free+0x10/0x10
[176614.896533] ? __pfx_compaction_alloc+0x10/0x10
[176614.900267] migrate_pages+0x39f/0x4a0
[176614.903627] ? __pfx_compaction_free+0x10/0x10
[176614.907235] ? __pfx_compaction_alloc+0x10/0x10
[176614.910867] compact_zone+0x2a6/0x5d0
[176614.914266] compact_node+0x8d/0xe0
[176614.917589] sysctl_compaction_handler+0x5d/0xb0
[176614.921326] proc_sys_call_handler+0x1d4/0x2f0
[176614.924983] proc_sys_write+0x13/0x20
[176614.928341] vfs_write+0x2ac/0x3d0
[176614.931624] ksys_write+0x67/0xf0
[176614.934816] __x64_sys_write+0x19/0x30
[176614.938189] do_syscall_64+0x59/0x90
[176614.941594] ? do_syscall_64+0x69/0x90
[176614.944991] ? __audit_syscall_exit+0xe1/0x130
[176614.948654] ? exit_to_user_mode_prepare+0x3b/0xd0
[176614.952463] ? syscall_exit_to_user_mode+0x38/0x60
[176614.956253] ? do_syscall_64+0x69/0x90
[176614.959628] ? do_syscall_64+0x69/0x90
[176614.962982] ? irqentry_exit+0x21/0x40
[176614.966380] ? exc_page_fault+0x95/0x190
[176614.969851] entry_SYSCALL_64_after_hwframe+0x6e/0xd8
[176614.973996] RIP: 0033:0x7f936c314887
[176614.981547] Code: 10 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
[176615.017280] RSP: 002b:00007ffe2c9db998 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
[176615.023885] RAX: ffffffffffffffda RBX: 0000000000000002 RCX: 00007f936c314887
[176615.030189] RDX: 0000000000000002 RSI: 0000564bf9ee6680 RDI: 0000000000000001
[176615.036749] RBP: 0000564bf9ee6680 R08: 0000000000000000 R09: 0000564bf9ee6680
[176615.043124] R10: 0000000000000077 R11: 0000000000000246 R12: 0000000000000002
[176615.049494] R13: 00007f936c41b780 R14: 00007f936c417600 R15: 00007f936c416a00
[176615.055958] </TASK>
[176615.058738] Modules linked in: cpuid xt_tcpudp rpcsec_gss_krb5 nfsv4 nfs fscache netfs tls xt_conntrack nft_chain_nat xt_MASQUERADE nf_nat nf_conntrack_netlink nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 xfrm_user xfrm_algo xt_addrtype nft_compat nf_tables libcrc32c nfnetlink br_netfilter bridge stp llc rpcrdma rdma_cm iw_cm ib_cm ib_core nvme_fabrics overlay binfmt_misc intel_rapl_msr intel_rapl_common intel_uncore_frequency_common isst_if_common nfit nls_iso8859_1 crct10dif_pclmul ppdev crc32_pclmul zfs(PO) polyval_clmulni polyval_generic ghash_clmulni_intel aesni_intel crypto_simd spl(O) cryptd rapl input_leds psmouse parport_pc ena i2c_piix4 serio_raw parport mac_hid dm_multipath scsi_dh_rdac scsi_dh_emc sch_fq_codel scsi_dh_alua nfsd auth_rpcgss nfs_acl lockd grace msr drm efi_pstore sunrpc ip_tables x_tables autofs4
[176615.105294] ---[ end trace 0000000000000000 ]---
[176615.108991] RIP: 0010:migrate_folio_extra+0x85/0x90
[176615.112772] Code: 31 ff 45 31 c0 c3 cc cc cc cc e8 06 d2 ff ff 44 89 f0 5b 41 5c 41 5d 41 5e 5d 31 d2 31 c9 31 f6 31 ff 45 31 c0 c3 cc cc cc cc <0f> 0b 66 0f 1f 84 00 00 00 00 00 90 90 90 90 90 90 90 90 90 90 90
[176615.125232] RSP: 0018:ffffb0c7f366b918 EFLAGS: 00010282
[176615.129129] RAX: 0157ffffc4008067 RBX: ffffdcdf963e9340 RCX: 0000000000000002
[176615.135501] RDX: ffffdcdf963e9340 RSI: ffffdce09cb61240 RDI: ffff98e202addcd0
[176615.141762] RBP: ffffb0c7f366b940 R08: 0000000000000000 R09: 0000000000000000
[176615.148218] R10: 0000000000000000 R11: 0000000000000000 R12: ffff98e202addcd0
[176615.154537] R13: ffffdce09cb61240 R14: 0000000000000002 R15: ffffb0c7f366b9c0
[176615.161043] FS: 00007f936c48c740(0000) GS:ffff943a72b00000(0000) knlGS:0000000000000000
[176615.167778] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[176615.171872] CR2: 00005574a10bb790 CR3: 000004976529a004 CR4: 00000000007706e0
[176615.178139] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[176615.184719] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[176615.191103] PKRU: 55555554

ProblemType: Bug
DistroRelease: Ubuntu 22.04
Package: linux-image-6.5.0-1011-aws 6.5.0-1011.11~22.04.1
ProcVersionSignature: Ubuntu 6.5.0-1011.11~22.04.1-aws 6.5.3
Uname: Linux 6.5.0-1011-aws x86_64
NonfreeKernelModules: zfs
ApportVersion: 2.20.11-0ubuntu82.5
Architecture: amd64
CasperMD5CheckResult: unknown
CloudArchitecture: x86_64
CloudID: aws
CloudName: aws
CloudPlatform: ec2
CloudRegion: us-east-1
CloudSubPlatform: metadata (http://169.254.169.254)
Date: Sun Feb 25 13:46:19 2024
Ec2AMI: ami-08c40ec9ead489470
Ec2AMIManifest: (unknown)
Ec2Architecture: x86_64
Ec2AvailabilityZone: us-east-1d
Ec2Imageid: ami-08c40ec9ead489470
Ec2InstanceType: u-12tb1.112xlarge
Ec2Instancetype: u-12tb1.112xlarge
Ec2Kernel: unavailable
Ec2Ramdisk: unavailable
Ec2Region: us-east-1
ProcEnviron:
 LC_CTYPE=C.UTF-8
 TERM=xterm-256color
 PATH=(custom, no user)
 LANG=C.UTF-8
 SHELL=/bin/bash
RebootRequiredPkgs: Error: path contained symlinks.
SourcePackage: linux-signed-aws-6.5
UpgradeStatus: No upgrade log present (probably fresh install)

[Ariel E (arielpagaya)]

Another time, on a system idle post reboot:

[21103.156799] ------------[ cut here ]------------
[21103.156806] kernel BUG at mm/migrate.c:654!
[21103.158475] invalid opcode: 0000 [#1] SMP NOPTI
[21103.160470] CPU: 400 PID: 570155 Comm: memclean Tainted: P O 6.5.0-1014-aws #14~22.04.1-Ubuntu
[21103.164199] Hardware name: Amazon EC2 u-12tb1.112xlarge/, BIOS 1.0 10/16/2017
[21103.166807] RIP: 0010:migrate_folio_extra+0x85/0x90
[21103.168837] Code: 31 ff 45 31 c0 c3 cc cc cc cc e8 16 d2 ff ff 44 89 f0 5b 41 5c 41 5d 41 5e 5d 31 d2 31 c9 31 f6 31 ff 45 31 c0 c3 cc cc cc cc <0f> 0b 66 0f 1f 84 00 00 00 00 00 90 90 90 90 90 90 90 90 90 90 90
[21103.175681] RSP: 0018:ffffb19895eeb8e8 EFLAGS: 00010282
[21103.177609] RAX: 0157ffffc0008025 RBX: ffffdbbe015d1380 RCX: 0000000000000002
[21103.180377] RDX: ffffdbbe015d1380 RSI: ffffdbbf0891d7c0 RDI: ffff95507dbbaba8
[21103.183038] RBP: ffffb19895eeb910 R08: 0000000000000000 R09: 0000000000000000
[21103.185760] R10: 0000000000000000 R11: 0000000000000000 R12: ffff95507dbbaba8
[21103.188524] R13: ffffdbbf0891d7c0 R14: 0000000000000002 R15: ffffb19895eeb990
[21103.191200] FS: 00007f194ba33740(0000) GS:ffff984972d00000(0000) knlGS:0000000000000000
[21103.194150] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[21103.196478] CR2: 00007f003f3f8ef8 CR3: 000001dc5161e003 CR4: 00000000007706e0
[21103.199112] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[21103.201748] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[21103.204479] PKRU: 55555554
[21103.207347] Call Trace:
[21103.210091] <TASK>
[21103.212786] ? show_regs+0x72/0x90
[21103.215881] ? die+0x38/0xb0
[21103.218718] ? do_trap+0xe3/0x100
[21103.221873] ? do_error_trap+0x75/0xb0
[21103.225130] ? migrate_folio_extra+0x85/0x90
[21103.228521] ? exc_invalid_op+0x53/0x80
[21103.231790] ? migrate_folio_extra+0x85/0x90
[21103.235131] ? asm_exc_invalid_op+0x1b/0x20
[21103.238489] ? migrate_folio_extra+0x85/0x90
[21103.242003] ? move_to_new_folio+0x145/0x160
[21103.245491] migrate_pages_batch+0x610/0x930
[21103.248935] ? __pfx_compaction_free+0x10/0x10
[21103.252382] ? __pfx_remove_migration_pte+0x10/0x10
[21103.256011] migrate_pages_sync+0x14e/0x200
[21103.259327] ? __pfx_compaction_free+0x10/0x10
[21103.262808] ? __pfx_compaction_alloc+0x10/0x10
[21103.266368] migrate_pages+0x39f/0x4a0
[21103.269723] ? __pfx_compaction_free+0x10/0x10
[21103.273247] ? __pfx_compaction_alloc+0x10/0x10
[21103.276776] compact_zone+0x2a6/0x5d0
[21103.279974] ? __flush_work.isra.0+0x21d/0x360
[21103.283451] compact_node+0x8d/0xe0
[21103.286633] sysctl_compaction_handler+0x5d/0xb0
[21103.290243] proc_sys_call_handler+0x1d4/0x2f0
[21103.293802] proc_sys_write+0x13/0x20
[21103.297048] vfs_write+0x2ac/0x3d0
[21103.300191] ksys_write+0x67/0xf0
[21103.303282] __x64_sys_write+0x19/0x30
[21103.306576] do_syscall_64+0x59/0x90
[21103.309858] ? audit_reset_context.part.0.constprop.0+0x290/0x300
[21103.313972] ? __audit_syscall_exit+0xe1/0x130
[21103.317507] ? exit_to_user_mode_prepare+0x3b/0xd0
[21103.321137] ? syscall_exit_to_user_mode+0x38/0x60
[21103.324764] ? do_syscall_64+0x69/0x90
[21103.327998] ? exc_...