arm64
-----

vm

	SERIES=noble
	ARCH=arm64
	DISK=${SERIES}_${ARCH}.qcow2

	wget https://cloud-images.ubuntu.com/$SERIES/current/${SERIES}-server-cloudimg-${ARCH}.img
	qemu-img create -F qcow2 -b ${SERIES}-server-cloudimg-${ARCH}.img -f qcow2 $DISK 8G

	sudo apt install -y qemu-system-arm qemu-efi

	dd if=/dev/zero of=flash1.img bs=1M count=64
	dd if=/dev/zero of=flash0.img bs=1M count=64
	dd if=/usr/share/qemu-efi/QEMU_EFI.fd of=flash0.img conv=notrunc

	qemu-system-aarch64 \
	-machine virt -cpu cortex-a57 \
	-pflash flash0.img -pflash flash1.img \
	\
	-smp cpus=2 -m 2048 \
	-nodefaults -no-user-config \
	-nographic -serial stdio \
	\
	-drive file=$DISK,if=none,id=drive0 \
	-device virtio-blk-device,drive=drive0 \
	\
	-drive file=test-cidata.iso,media=cdrom \
	\
	-netdev user,hostfwd=::22222-:22,id=net0 \
	-device virtio-net-device,netdev=net0


	$ ssh ubuntu@127.0.0.1 -p 22222

	$ lsb_release -cs
	No LSB modules are available.
	noble
	
	$ uname -m
	aarch64

	$ uname -rv
	6.5.0-9-generic #9-Ubuntu SMP PREEMPT_DYNAMIC Fri Oct  6 20:34:49 UTC 2023

kdump-tools

	sudo apt update && sudo apt install -y linux-crashdump # No, Yes
	sudo sed 's/crashkernel=[^" ]\+/crashkernel=512M/' -i /etc/default/grub.d/kdump-tools.cfg
	sudo update-grub && sudo reboot

	$ sudo dmesg | grep 'crashkernel reserved:'
	[    0.000000] crashkernel reserved: 0x0000000091600000 - 0x00000000b1600000 (512 MB)

	$ sudo kdump-config status
	 * Invalid symlink : /var/lib/kdump/initrd.img
	 * Invalid symlink : /var/lib/kdump/vmlinuz
	current state   : Not ready to kdump

	$ sudo kdump-config reload
	 * unloaded kdump kernel
	 * Creating symlink /var/lib/kdump/vmlinuz
	kdump-tools: Generating /var/lib/kdump/initrd.img-6.5.0-9-generic
	 * Creating symlink /var/lib/kdump/initrd.img
	 * loaded kdump kernel

	$ sudo kdump-config status
	current state   : ready to kdump

crashdump

	$ echo c | sudo tee /proc/sysrq-trigger

	$ find /var/crash
	/var/crash
	/var/crash/202311081722
	/var/crash/202311081722/dmesg.202311081722
	/var/crash/202311081722/dump.202311081722
	/var/crash/kexec_cmd
	/var/crash/kdump_lock
	/var/crash/linux-image-6.5.0-9-generic-202311081722.crash

debug symbols

	dpkg -l | awk '$2 ~ /linux-image-[0-9.-]+-generic/ { print $2, $3}' \
	  | while read pkg version; do \
	      dbgpkg="linux-image-unsigned-${pkg#linux-image-}-dbgsym"; \
	      arch=$(dpkg --print-architecture); \
	      wget "https://launchpad.net/ubuntu/+archive/primary/+files/${dbgpkg}_${version}_${arch}.ddeb"; \
	    done

	ar p linux-image-unsigned-*-dbgsym_*.ddeb data.tar.xz | tar xJ --wildcards './usr/lib/debug/boot/vmlinux-*-generic'

crash (old)

	$ dpkg -s crash | grep Version:
	Version: 8.0.2-1ubuntu1

	$ crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	...
	crash: invalid structure member offset: module_core_size
	       FILE: kernel.c  LINE: 3781  FUNCTION: module_init()

	[/usr/bin/crash] error trace: aaaac5c23f9c => aaaac58f2f84 => aaaac5974d3c => aaaac59e7330

crash (new)

	sudo add-apt-repository -y ppa:mfo/noble-crash-v2 && sudo apt install -y crash

	$ dpkg -s crash | grep Version:
	Version: 8.0.3+ds1-3ubuntu1

	$ crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	...
	WARNING: active task ffff000041b7e480 on cpu 0 not found in PID hash
	WARNING: active task ffff000040104300 on cpu 1 not found in PID hash

	      KERNEL: ./usr/lib/debug/boot/vmlinux-6.5.0-9-generic
	    DUMPFILE: /var/crash/202311081722/dump.202311081722  [PARTIAL DUMP]
		CPUS: 2
		DATE: Wed Nov  8 17:17:27 UTC 2023
	      UPTIME: 01:25:18
	LOAD AVERAGE: 0.27, 0.11, 0.16
	       TASKS: 4
	    NODENAME: test
	     RELEASE: 6.5.0-9-generic
	     VERSION: #9-Ubuntu SMP PREEMPT_DYNAMIC Fri Oct  6 20:34:49 UTC 2023
	     MACHINE: aarch64  (unknown Mhz)
	      MEMORY: 2.1 GB
	       PANIC: "Kernel panic - not syncing: sysrq triggered crash"
		 PID: 5111
	     COMMAND: "tee"
		TASK: ffff000041b7e480  [THREAD_INFO: ffff000041b7e480]
		 CPU: 0
	       STATE: TASK_RUNNING (PANIC)

	crash> bt
	PID: 5111     TASK: ffff000041b7e480  CPU: 0    COMMAND: "tee"
	 #0 [ffff80008404b8d0] machine_kexec at ffff800080182cf0
	 #1 [ffff80008404ba60] __crash_kexec at ffff80008033b2d8
	 #2 [ffff80008404bae0] panic at ffff800080206920
	 #3 [ffff80008404bb70] sysrq_handle_crash at ffff800080e4d65c
	 #4 [ffff80008404bb80] __handle_sysrq at ffff800080e4e420
	 #5 [ffff80008404bbf0] write_sysrq_trigger at ffff800080e4efb8
	 #6 [ffff80008404bc10] proc_reg_write at ffff8000806fad50
	 #7 [ffff80008404bc90] vfs_write at ffff8000806246b0
	 #8 [ffff80008404bce0] ksys_write at ffff800080624b2c
	 #9 [ffff80008404bd10] __arm64_sys_write at ffff800080624c14
	#10 [ffff80008404be00] invoke_syscall at ffff800080173d30
	#11 [ffff80008404be30] el0_svc_common.constprop.0 at ffff800080173e38
	#12 [ffff80008404be60] do_el0_svc at ffff800080173f7c
	#13 [ffff80008404be80] el0_svc at ffff80008167d934
	#14 [ffff80008404bea0] el0t_64_sync_handler at ffff80008167dfdc
	#15 [ffff80008404bfe0] el0t_64_sync at ffff800080151e44
	...
	crash> quit