amd64
-----

vm

	lxc launch --vm -c limits.cpu=2 -c limits.memory=2GiB ubuntu-daily:noble noble-vm
	lxc shell noble-vm

	# uname -rv
	6.5.0-9-generic #9-Ubuntu SMP PREEMPT_DYNAMIC Sat Oct  7 01:35:40 UTC 2023


kdump-tools

	sudo apt update && apt install -y linux-crashdump # No, Yes
	sudo sed 's/crashkernel=[^" ]\+/crashkernel=512M/' -i /etc/default/grub.d/kdump-tools.cfg
	sudo update-grub && sudo reboot

	...

	# dmesg | grep 'crashkernel.*RAM'
	[    0.018756] Reserving 512MB of memory at 816MB for crashkernel (System RAM: 2043MB)

	# kdump-config status
	current state   : ready to kdump

crashdump

	# echo c >/proc/sysrq-trigger

	...

	# find /var/crash/
	/var/crash/
	/var/crash/kdump_lock
	/var/crash/kexec_cmd
	/var/crash/202311081257
	/var/crash/202311081257/dmesg.202311081257
	/var/crash/202311081257/dump.202311081257
	/var/crash/linux-image-6.5.0-9-generic-202311081257.crash

debug symbols

	dpkg -l | awk '$2 ~ /linux-image-[0-9.-]+-generic/ { print $2, $3}' \
	  | while read pkg version; do \
	      dbgpkg="linux-image-unsigned-${pkg#linux-image-}-dbgsym"; \
	      arch=$(dpkg --print-architecture); \
	      wget "https://launchpad.net/ubuntu/+archive/primary/+files/${dbgpkg}_${version}_${arch}.ddeb"; \
	    done

	ar p linux-image-unsigned-*-dbgsym_*.ddeb data.tar.xz | tar xJ --wildcards './usr/lib/debug/boot/vmlinux-*-generic'

crash (old)

	# dpkg -s crash | grep Version:
	Version: 8.0.2-1ubuntu1

	# crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	...
	crash: invalid structure member offset: module_core_size
	       FILE: kernel.c  LINE: 3781  FUNCTION: module_init()

	[/usr/bin/crash] error trace: 558bf515829f => 558bf4dd7efc => 558bf4e64bff => 558bf4eef301

crash (new)

	sudo add-apt-repository -y ppa:mfo/noble-crash-v2 && sudo apt install -y crash

	# crash ./usr/lib/debug/boot/vmlinux-*-generic /var/crash/*/dump.*
	...
	please wait... (determining panic task)
	WARNING: active task ffff93c084ed0000 on cpu 0 not found in PID hash

	      KERNEL: ./usr/lib/debug/boot/vmlinux-6.5.0-9-generic
	    DUMPFILE: /var/crash/202311081257/dump.202311081257  [PARTIAL DUMP]
		CPUS: 2
		DATE: Wed Nov  8 12:57:51 UTC 2023
	      UPTIME: 00:01:42
	LOAD AVERAGE: 0.28, 0.18, 0.07
	       TASKS: 3
	    NODENAME: noble-vm
	     RELEASE: 6.5.0-9-generic
	     VERSION: #9-Ubuntu SMP PREEMPT_DYNAMIC Sat Oct  7 01:35:40 UTC 2023
	     MACHINE: x86_64  (2495 Mhz)
	      MEMORY: 2 GB
	       PANIC: "Kernel panic - not syncing: sysrq triggered crash"
		 PID: 4488
	     COMMAND: "bash"
		TASK: ffff93c084ed0000  [THREAD_INFO: ffff93c084ed0000]
		 CPU: 0
	       STATE: TASK_RUNNING (PANIC)

	crash> bt
	PID: 4488     TASK: ffff93c084ed0000  CPU: 0    COMMAND: "bash"
	 #0 [ffffa9c500d0bb38] machine_kexec at ffffffff9dcafa3b
	 #1 [ffffa9c500d0bb98] __crash_kexec at ffffffff9de133f3
	 #2 [ffffa9c500d0bc60] panic at ffffffff9dcf60e4
	 #3 [ffffa9c500d0bce0] sysrq_handle_crash at ffffffff9e6a4b5a
	 #4 [ffffa9c500d0bcf0] __handle_sysrq at ffffffff9e6a52e3
	 #5 [ffffa9c500d0bd38] write_sysrq_trigger at ffffffff9e6a5b28
	 #6 [ffffa9c500d0bd50] proc_reg_write at ffffffff9e1691b9
	 #7 [ffffa9c500d0bd70] vfs_write at ffffffff9e0af87f
	 #8 [ffffa9c500d0be10] ksys_write at ffffffff9e0aff83
	 #9 [ffffa9c500d0be50] __x64_sys_write at ffffffff9e0b0039
	#10 [ffffa9c500d0be60] do_syscall_64 at ffffffff9ed399a9
	#11 [ffffa9c500d0bf50] entry_SYSCALL_64_after_hwframe at ffffffff9ee000e6
	    RIP: 00007fd7ba71b214  RSP: 00007ffd1900fbc8  RFLAGS: 00000202
	    RAX: ffffffffffffffda  RBX: 0000000000000002  RCX: 00007fd7ba71b214
	    RDX: 0000000000000002  RSI: 000055b4326b7d60  RDI: 0000000000000001
	    RBP: 000055b4326b7d60   R8: 0000000000000073   R9: 0000000000000000
	    R10: 0000000000000000  R11: 0000000000000202  R12: 0000000000000002
	    R13: 00007fd7ba7ff7a0  R14: 00007fd7ba7fd120  R15: 0000000000000000
	    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b
	crash> quit