Synthetic reproducer on Focal
===

Ubuntu Server cloud image:
---

SERIES=focal

wget https://cloud-images.ubuntu.com/$SERIES/current/${SERIES}-server-cloudimg-amd64.img
qemu-img create -F qcow2 -b ${SERIES}-server-cloudimg-amd64.img -f qcow2 $SERIES.qcow2 8G


cloud-init image:
---

echo 'local-hostname: test' >meta-data

PUBKEY=$(cat $HOME/.ssh/id_rsa.pub)

cat <<EOF >user-data
#cloud-config
users:
  - name: ubuntu
    ssh-authorized-keys: ["$PUBKEY"]
    sudo: ['ALL=(ALL) NOPASSWD:ALL']
    groups: sudo
    shell: /bin/bash
runcmd:
   - echo 'AllowUsers ubuntu' >>/etc/ssh/sshd_config
   - restart ssh
EOF

genisoimage -output test-cidata.iso -volid cidata -joliet -rock user-data meta-data


QEMU VM:
---
- multipath disk on virtio-scsi:
- host forwarding for ssh login:

truncate -s 1G multipath.img

qemu-system-x86_64 \
  -machine q35 -accel kvm -cpu host -m 2048 \
  -nodefaults -no-user-config \
  -nographic -serial stdio \
  \
  -net nic -net user,hostfwd=::22222-:22 \
  \
  -drive file=$SERIES.qcow2 \
  -drive file=test-cidata.iso,media=cdrom \
  \
  -device virtio-scsi,id=scsi1 \
  \
  -drive file=multipath.img,if=none,cache=none,format=file,locking=off,id=path0 \
  -device scsi-hd,bus=scsi1.0,serial=helloworld,drive=path0 \
  \
  -drive file=multipath.img,if=none,cache=none,format=file,locking=off,id=path1 \
  -device scsi-hd,bus=scsi1.0,serial=helloworld,drive=path1

$ ssh ubuntu@127.0.0.1 -p 22222

$ sudo multipath -l
mpatha (0QEMU_QEMU_HARDDISK_helloworld) dm-0 QEMU,QEMU HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 0:0:0:0 sda 8:0  active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 0:0:1:0 sdb 8:16 active undef running

Latest version:
---

sudo add-apt-repository -y 'deb http://archive.ubuntu.com/ubuntu focal-proposed main'
sudo apt install -y multipath-tools/focal-proposed

sudo systemctl restart multipathd.service

$ dpkg -s multipath-tools | grep Version:
Version: 0.8.3-1ubuntu2.2


Debug symbols
---

V1=$(dpkg-query -Wf '${Version}' multipath-tools)
V2=$(dpkg-query -Wf '${Version}' libdevmapper1.02.1 | cut -d: -f2-)

wget https://launchpad.net/ubuntu/+archive/primary/+files/multipath-tools-dbgsym_${V1}_amd64.ddeb
wget https://launchpad.net/ubuntu/+archive/primary/+files/libdevmapper1.02.1-dbgsym_${V2}_amd64.ddeb

sudo dpkg -i ./multipath-tools-dbgsym_${V1}_amd64.ddeb ./libdevmapper1.02.1-dbgsym_${V2}_amd64.ddeb


Debugger
---

sudo apt install -y gdb

sudo gdb -p $(pidof multipathd)

(gdb) b dm_get_map
Breakpoint 1 at ...
(gdb) c
Thread 5 "multipathd" hit Breakpoint 1, dm_get_map (name=0x558d02dc6880 "mpatha", ...

(gdb) b dm_get_next_target
Breakpoint 2 at ...
(gdb) c
Thread 5 "multipathd" hit Breakpoint 2, dm_get_next_target (...

(gdb) finish
Run till exit from #0  dm_get_next_target (...
Value returned is $1 = (void *) 0x0

(gdb) p params
$2 = 0x7fef90004db0 "0 0 2 1 service-time 0 1 2 8:0 1 1 service-time 0 1 2 8:16 1 1 "
(gdb) set params = 0
(gdb) p params
$3 = 0x0

Now, NOTE the value returned from both functions 
(ZERO, i.e., "OK", even though empty map parameters is invalid)

(gdb) finish
Run till exit from #0  dm_get_map (name=0x563ca34f5f70 "mpatha", ...
Value returned is $4 = 0

(gdb) p params
$5 = "(null)", '\000' <repeats 4089 times>

(gdb) finish
Run till exit from #0  0x00007f28d78b0911 in update_multipath_table (...

...
Value returned is $6 = 0


Now, the multipath map state has NULL in all parameters:

(gdb) b disassemble_map
(gdb) c

Thread 5 "multipathd" hit Breakpoint 3, disassemble_map (...
(gdb) finish

(gdb) p mpp.features
$8 = 0x7f2e80000e40 "(null)"

(gdb) p mpp.hwhandler
$10 = 0x7f2e80000f40 "(null)"

(gdb) p mpp.pg
$11 = (vector) 0x0

(gdb) p mpp.selector
$12 = (vector) 0x0

(gdb) q


Compare with a normal run, where params is not NULL:

(gdb) p mpp.features
$2 = 0x55f7670f3940 "0"

(gdb) p mpp.hwhandler
$3 = 0x55f7670f8500 "0"

(gdb) p *mpp.pg
$4 = {allocated = 2, slot = 0x55f7670f3990}

(gdb) p mpp.selector
$5 = 0x55f7670f9e00 "service-time 0"