s390x
-----

vm

	sudo apt install -y qemu-system-s390x

	qemu-system-s390x \
	  -machine s390-ccw-virtio -cpu qemu \
	\
	-smp cpus=2 -m 2048 \
	-nodefaults -no-user-config \
	-nographic -serial stdio \
	\
	-drive file=$DISK,if=none,id=drive0 \
	-device virtio-blk,drive=drive0 \
	\
	-drive file=test-cidata.iso,media=cdrom \
	\
	-netdev user,hostfwd=::22224-:22,id=net0 \
	-device virtio-net,netdev=net0

	$ ssh ubuntu@127.0.0.1 -p 22224

	$ lsb_release -cs
	No LSB modules are available.
	noble

	$ uname -m
	s390x

	$ uname -rv
	6.5.0-9-generic #9-Ubuntu SMP Fri Oct  6 19:43:35 UTC 2023

kdump-tools

	sudo apt update && sudo apt install -y linux-crashdump # No, Yes
	sudo sed '/^parameters =/ s/$/ crashkernel=512M/' -i /etc/zipl.conf
	sudo zipl && sudo reboot

	$ sudo dmesg | grep 'crashkernel.*RAM'
	[   26.793337] setup: Reserving 512MB of memory at 1510MB for crashkernel (System RAM: 1536MB)

	$ sudo kdump-config status
	current state   : ready to kdump

crashdump

	$ echo c | sudo tee /proc/sysrq-trigger

	$ find /var/crash
	/var/crash
	/var/crash/linux-image-6.5.0-9-generic-202311081631.crash
	/var/crash/202311081631
	/var/crash/202311081631/dump.202311081631
	/var/crash/202311081631/dmesg.202311081631
	/var/crash/kexec_cmd
	/var/crash/kdump_lock

debug symbols

	dpkg -l | awk '$2 ~ /linux-image-[0-9.-]+-generic/ { print $2, $3}' \
	  | while read pkg version; do \
	      dbgpkg="linux-image-unsigned-${pkg#linux-image-}-dbgsym"; \
	      arch=$(dpkg --print-architecture); \
	      wget "https://launchpad.net/ubuntu/+archive/primary/+files/${dbgpkg}_${version}_${arch}.ddeb"; \
	    done

	ar p linux-image-unsigned-*-dbgsym_*.ddeb data.tar.xz | tar xJ --wildcards './usr/lib/debug/boot/vmlinux-*-generic'

crash (old)

	# dpkg -s crash | grep Version:
	Version: 8.0.2-1ubuntu1

	# crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	crash: invalid structure member offset: module_core_size
	       FILE: kernel.c  LINE: 3781  FUNCTION: module_init()

	[/usr/bin/crash] error trace: 2aa1be60c84 => 2aa1bb029f2 => 2aa1bb951b2 => 2aa1bc0d668

crash (new)

	sudo add-apt-repository -y ppa:mfo/noble-crash-v2 && sudo apt install -y crash

	# dpkg -s crash | grep Version:
	Version: 8.0.3+ds1-3ubuntu1

	$ crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	...
	WARNING: active task 3833600 on cpu 1 not found in PID hash

	      KERNEL: ./usr/lib/debug/boot/vmlinux-6.5.0-9-generic
	    DUMPFILE: /var/crash/202311081631/dump.202311081631  [PARTIAL DUMP]
		CPUS: 2
		DATE: Wed Nov  8 16:22:13 UTC 2023
	      UPTIME: 00:23:37
	LOAD AVERAGE: 0.02, 0.57, 1.58
	       TASKS: 3
	    NODENAME: test
	     RELEASE: 6.5.0-9-generic
	     VERSION: #9-Ubuntu SMP Fri Oct  6 19:43:35 UTC 2023
	     MACHINE: s390x  (unknown Mhz)
	      MEMORY: 2 GB
	       PANIC: "Kernel panic - not syncing: sysrq triggered crash"
		 PID: 4610
	     COMMAND: "tee"
		TASK: 3833600  [THREAD_INFO: 3833600]
		 CPU: 1
	       STATE: TASK_RUNNING (PANIC)

	crash> bt
	...
	 #0 [380002179a8] smp_call_ipl_cpu at 11e2fe
	 #1 [380002179c8] __crash_kexec at 2adfaa
	 #2 [38000217ac0] panic at 1a8208
	 #3 [38000217b68] sysrq_reset_seq_param_set at a3f860
	 #4 [38000217b80] __handle_sysrq at a4055e
	 #5 [38000217bf8] write_sysrq_trigger at a4100c
	 #6 [38000217c30] proc_reg_write at 60dc9e
	 #7 [38000217c80] vfs_write at 5470d4
	 #8 [38000217d40] ksys_write at 547600
	 #9 [38000217d90] __do_syscall at e77730
	#10 [38000217e98] system_call at e89b58
	...
	crash> quit