Ubuntu 18.04.3 - Bug 176932 1. Boot and test with cap-ibs=workaround,cap-ccf-assist=on kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=on -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) #dmesg | grep cache-flush [ 0.000000] count-cache-flush: hardware assisted flush sequence enabled root@ubuntu:~# dmesg -C root@ubuntu:~# echo 0 > /sys/kernel/debug/powerpc/count_cache_flush root@ubuntu:~# dmesg [ 100.068718] count-cache-flush: software flush disabled. root@ubuntu:~# echo 1 > /sys/kernel/debug/powerpc/count_cache_flush root@ubuntu:~# dmesg [ 100.068718] count-cache-flush: software flush disabled. [ 107.769047] count-cache-flush: hardware assisted flush sequence enabled root@ubuntu:~# add xmon=on kernel cmdline #echo 1 > /proc/sys/kernel/sysrq #echo c > /proc/sysrq-trigger 6:mon> di $_switch 20 c00000000000db00 7c0802a6 mflr r0 c00000000000db04 f8010010 std r0,16(r1) c00000000000db08 f821fe31 stdu r1,-464(r1) c00000000000db0c f9c100e0 std r14,224(r1) c00000000000db10 f9e100e8 std r15,232(r1) c00000000000db14 fa0100f0 std r16,240(r1) c00000000000db18 fa2100f8 std r17,248(r1) c00000000000db1c fa410100 std r18,256(r1) c00000000000db20 fa610108 std r19,264(r1) c00000000000db24 fa810110 std r20,272(r1) c00000000000db28 faa10118 std r21,280(r1) c00000000000db2c fac10120 std r22,288(r1) c00000000000db30 fae10128 std r23,296(r1) c00000000000db34 fb010130 std r24,304(r1) c00000000000db38 fb210138 std r25,312(r1) c00000000000db3c fb410140 std r26,320(r1) c00000000000db40 fb610148 std r27,328(r1) c00000000000db44 fb810150 std r28,336(r1) c00000000000db48 fba10158 std r29,344(r1) c00000000000db4c fbc10160 std r30,352(r1) c00000000000db50 fbe10168 std r31,360(r1) c00000000000db54 f8010170 std r0,368(r1) c00000000000db58 7ee00026 mfcr r23 c00000000000db5c fae101a0 std r23,416(r1) c00000000000db60 f8230000 std r1,0(r3) c00000000000db64 4bffdb1d bl c00000000000b680 # flush_count_cache+0x0/0x2480 -----------------------OK c00000000000db68 3cc06000 lis r6,24576 c00000000000db6c 7d40322c dcbt 0,r6,10 c00000000000db70 38c4f4d0 addi r6,r4,-2864 c00000000000db74 f8cd0260 std r6,608(r13) c00000000000db78 e9040000 ld r8,0(r4) c00000000000db7c 48000064 b c00000000000dbe0 # _switch+0xe0/0x180 6:mon> di $flush_count_cache 4d c00000000000b680 7d2802a6 mflr r9 c00000000000b684 48000005 bl c00000000000b688 # flush_count_cache+0x8/0x2480 ... c00000000000b784 4800001c b c00000000000b7a0 # flush_count_cache+0x120/0x2480 c00000000000b788 60000000 nop ... c00000000000b7a0 7d2803a6 mtlr r9 c00000000000b7a4 39207fff li r9,32767 c00000000000b7a8 7d2903a6 mtctr r9 c00000000000b7ac 4c400420 bcctr- 2,lt c00000000000b7b0 4e800020 blr ------------------------------------OK 6:mon> 2. Boot and test with cap-ibs=workaround,cap-ccf-assist=off #kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=off -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) #uname -a Linux ubuntu 4.15.0-59-generic #66-Ubuntu SMP Wed Aug 14 10:56:59 UTC 2019 ppc64le ppc64le ppc64le GNU/Linux root@ubuntu:~# dmesg | grep cache-flush [ 0.000000] count-cache-flush: full software flush sequence enabled. #kvm -M pseries-bionic-sxxm,cap-ibs=workaround -monitor stdio -vga none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 dmesg | grep cache-flush [ 0.000000] count-cache-flush: full software flush sequence enabled. add xmon=on kernel cmdline #echo 1 > /proc/sys/kernel/sysrq #echo c > /proc/sysrq-trigger 6:mon> di $_switch 20 c00000000000db00 7c0802a6 mflr r0 c00000000000db04 f8010010 std r0,16(r1) c00000000000db08 f821fe31 stdu r1,-464(r1) c00000000000db0c f9c100e0 std r14,224(r1) c00000000000db10 f9e100e8 std r15,232(r1) c00000000000db14 fa0100f0 std r16,240(r1) c00000000000db18 fa2100f8 std r17,248(r1) c00000000000db1c fa410100 std r18,256(r1) c00000000000db20 fa610108 std r19,264(r1) c00000000000db24 fa810110 std r20,272(r1) c00000000000db28 faa10118 std r21,280(r1) c00000000000db2c fac10120 std r22,288(r1) c00000000000db30 fae10128 std r23,296(r1) c00000000000db34 fb010130 std r24,304(r1) c00000000000db38 fb210138 std r25,312(r1) c00000000000db3c fb410140 std r26,320(r1) c00000000000db40 fb610148 std r27,328(r1) c00000000000db44 fb810150 std r28,336(r1) c00000000000db48 fba10158 std r29,344(r1) c00000000000db4c fbc10160 std r30,352(r1) c00000000000db50 fbe10168 std r31,360(r1) c00000000000db54 f8010170 std r0,368(r1) c00000000000db58 7ee00026 mfcr r23 c00000000000db5c fae101a0 std r23,416(r1) c00000000000db60 f8230000 std r1,0(r3) c00000000000db64 4bffdb1d bl c00000000000b680 # flush_count_cache+0x0/0x2480 ------------------OK c00000000000db68 3cc06000 lis r6,24576 c00000000000db6c 7d40322c dcbt 0,r6,10 c00000000000db70 38c4f4d0 addi r6,r4,-2864 c00000000000db74 f8cd0260 std r6,608(r13) c00000000000db78 e9040000 ld r8,0(r4) c00000000000db7c 48000064 b c00000000000dbe0 # _switch+0xe0/0x180 6:mon> di $flush_count_cache 4d c00000000000b680 7d2802a6 mflr r9 c00000000000b684 48000005 bl c00000000000b688 # flush_count_cache+0x8/0x2480 ... c00000000000b784 4800001c b c00000000000b7a0 # flush_count_cache+0x120/0x2480 c00000000000b788 60000000 nop ... c00000000000b7a0 7d2803a6 mtlr r9 c00000000000b7a4 39207fff li r9,32767 c00000000000b7a8 7d2903a6 mtctr r9 c00000000000b7ac 4c400420 bcctr- 2,lt c00000000000b7b0 60000000 nop --------------------------------------------------OK 3. Boot and test with cap-ibs=broken # kvm -M pseries-bionic-sxxm,cap-ibs=broken -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) # dmesg | grep cache-flush [ 0.000000] count-cache-flush: software flush disabled. add xmon=on kernel cmdline #echo 1 > /proc/sys/kernel/sysrq #echo c > /proc/sysrq-trigger 6:mon> di $_switch 20 c00000000000db00 7c0802a6 mflr r0 c00000000000db04 f8010010 std r0,16(r1) c00000000000db08 f821fe31 stdu r1,-464(r1) c00000000000db0c f9c100e0 std r14,224(r1) c00000000000db10 f9e100e8 std r15,232(r1) c00000000000db14 fa0100f0 std r16,240(r1) c00000000000db18 fa2100f8 std r17,248(r1) c00000000000db1c fa410100 std r18,256(r1) c00000000000db20 fa610108 std r19,264(r1) c00000000000db24 fa810110 std r20,272(r1) c00000000000db28 faa10118 std r21,280(r1) c00000000000db2c fac10120 std r22,288(r1) c00000000000db30 fae10128 std r23,296(r1) c00000000000db34 fb010130 std r24,304(r1) c00000000000db38 fb210138 std r25,312(r1) c00000000000db3c fb410140 std r26,320(r1) c00000000000db40 fb610148 std r27,328(r1) c00000000000db44 fb810150 std r28,336(r1) c00000000000db48 fba10158 std r29,344(r1) c00000000000db4c fbc10160 std r30,352(r1) c00000000000db50 fbe10168 std r31,360(r1) c00000000000db54 f8010170 std r0,368(r1) c00000000000db58 7ee00026 mfcr r23 c00000000000db5c fae101a0 std r23,416(r1) c00000000000db60 f8230000 std r1,0(r3) c00000000000db64 60000000 nop c00000000000db68 3cc06000 lis r6,24576 c00000000000db6c 7d40322c dcbt 0,r6,10 c00000000000db70 38c4f4d0 addi r6,r4,-2864 c00000000000db74 f8cd0260 std r6,608(r13) c00000000000db78 e9040000 ld r8,0(r4) c00000000000db7c 48000064 b c00000000000dbe0 # _switch+0xe0/0x180 6:mon> 6:mon> di $flush_count_cache 4d c00000000000b680 7d2802a6 mflr r9 c00000000000b684 48000005 bl c00000000000b688 # flush_count_cache+0x8/0x2480 ... c00000000000b784 4800001c b c00000000000b7a0 # flush_count_cache+0x120/0x2480 c00000000000b788 60000000 nop ... c00000000000b7a0 7d2803a6 mtlr r9 c00000000000b7a4 39207fff li r9,32767 c00000000000b7a8 7d2903a6 mtctr r9 c00000000000b7ac 4c400420 bcctr- 2,lt c00000000000b7b0 60000000 nop 6:mon> 4. Test Migration: Source: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) info status VM status: running (qemu) migrate -d tcp:localhost:4444 (qemu) info migrate globals: store-global-state=1, only_migratable=0, send-configuration=1, send-section-footer=1 capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off postcopy-ram: off x-colo: off release-ram: off block: off return-path: off pause-before-switchover: off x-multifd: off Migration status: completed total time: 19681 milliseconds downtime: 70 milliseconds setup: 25 milliseconds transferred ram: 648880 kbytes throughput: 271.95 mbps remaining ram: 0 kbytes total ram: 8405056 kbytes duplicate: 1958715 pages skipped: 0 pages normal: 157608 pages normal bytes: 630432 kbytes dirty sync count: 6 page size: 4 kbytes (qemu) info status VM status: paused (postmigrate) (qemu) Destination: # kvm -M pseries-bionic-sxxm,cap-ibs=broken -monitor stdio -vnc none -nographic -serial /dev/pts/5 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 -incoming tcp:0:4444 QEMU 2.11.1 monitor - type 'help' for more information (qemu) info status VM status: paused (inmigrate) (qemu) qemu-system-ppc64: cap-ibs higher level (1) in incoming stream than on destination (0) qemu-system-ppc64: error while loading state for instance 0x0 of device 'spapr' qemu-system-ppc64: load of migration failed: Invalid argument --------- Source: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=on -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) migrate -d tcp:localhost:4444 (qemu) info status VM status: paused (postmigrate) (qemu) Destination: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=off -monitor stdio -vnc none -nographic -serial /dev/pts/5 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 -incoming tcp:0:4444 QEMU 2.11.1 monitor - type 'help' for more information (qemu) info status VM status: paused (inmigrate) (qemu) qemu-system-ppc64: cap-ccf-assist higher level (1) in incoming stream than on destination (0) qemu-system-ppc64: error while loading state for instance 0x0 of device 'spapr' qemu-system-ppc64: load of migration failed: Invalid argument ---------- Source: # kvm -M pseries-bionic-sxxm,cap-ibs=broken -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) migrate -d tcp:localhost:4444 (qemu) info status VM status: running (qemu) info status VM status: paused (postmigrate) (qemu) Destination: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=on -monitor stdio -vnc none -nographic -serial /dev/pts/5 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 -incoming tcp:0:4444 QEMU 2.11.1 monitor - type 'help' for more information (qemu) info status VM status: paused (inmigrate) (qemu) qemu-system-ppc64: warning: cap-ibs lower level (0) in incoming stream than on destination (1) qemu-system-ppc64: warning: cap-ccf-assist lower level (0) in incoming stream than on destination (1) (qemu) info status VM status: running (qemu) root@ubuntu:~# dmesg | grep cache-flush [ 0.000000] count-cache-flush: software flush disabled. root@ubuntu:~# echo 1 > /sys/kernel/debug/powerpc/count_cache_flush root@ubuntu:~# dmesg | grep cache-flush [ 0.000000] count-cache-flush: software flush disabled. [ 123.667140] count-cache-flush: software flush disabled. root@ubuntu:~# reboot #dmesg | grep cache-flush [ 0.000000] count-cache-flush: hardware assisted flush sequence enabled root@ubuntu:~# dmesg -C root@ubuntu:~# echo 0 > /sys/kernel/debug/powerpc/count_cache_flush root@ubuntu:~# dmesg [ 86.022822] count-cache-flush: software flush disabled. root@ubuntu:~# ----------------- Source: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=off -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information ~# cat /sys/kernel/debug/powerpc/count_cache_flush 1 root@ubuntu:~# dmesg|grep cache-flush [ 0.000000] count-cache-flush: full software flush sequence enabled. root@ubuntu:~# (qemu) migrate -d tcp:localhost:4444 (qemu) info status VM status: paused (postmigrate) (qemu) Destination: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=on -monitor stdio -vnc none -nographic -serial /dev/pts/5 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 -incoming tcp:0:4444 QEMU 2.11.1 monitor - type 'help' for more information (qemu) infqemu-system-ppc64: warning: cap-ccf-assist lower level (0) in incoming stream than on destination (1) info status VM status: running (qemu) # dmesg | grep cache-flush [ 0.000000] count-cache-flush: full software flush sequence enabled. [ 325.702015] count-cache-flush: software flush disabled. root@ubuntu:~# echo 1 > /sys/kernel/debug/powerpc/count_cache_flush root@ubuntu:~# dmesg | grep cache-flush [ 0.000000] count-cache-flush: full software flush sequence enabled. [ 325.702015] count-cache-flush: software flush disabled. [ 336.214800] count-cache-flush: full software flush sequence enabled. #reboot #dmesg | grep cache-flush [ 0.000000] count-cache-flush: hardware assisted flush sequence enabled ---------- Source: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=on -monitor stdio -vnc none -nographic -serial /dev/pts/0 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 QEMU 2.11.1 monitor - type 'help' for more information (qemu) info status VM status: running (qemu) migrate -d tcp:localhost:4444 (qemu) info status VM status: paused (postmigrate) (qemu) Destination: # kvm -M pseries-bionic-sxxm,cap-ibs=workaround,cap-ccf-assist=off -monitor stdio -vnc none -nographic -serial /dev/pts/5 -smp 8 -m 8192 /home/sath/ubuntu-18.04-ppc64le.qcow2 -incoming tcp:0:4444 QEMU 2.11.1 monitor - type 'help' for more information (qemu) qemu-system-ppc64: cap-ccf-assist higher level (1) in incoming stream than on destination (0) qemu-system-ppc64: error while loading state for instance 0x0 of device 'spapr' qemu-system-ppc64: load of migration failed: Invalid argument