At frame4 (unix_stream_connect) we poked at some random 0xffff addresses until we found the right one:
crash> ptype struct sockaddr_un
type = struct sockaddr_un {
__kernel_sa_family_t sun_family;
char sun_path[108];
}
So we started looking at the filesystem
crash> mod -s xfs
MODULE NAME SIZE OBJECT FILE
ffffffffc0652600 xfs 1511424 /tmp/kernel/usr/lib/debug/lib/modules/4.18.0-240.10.1.el8_3.x86_64/kernel/fs/xfs/xfs.ko.debug
crash> mount
MOUNT SUPERBLK TYPE DEVNAME DIRNAME
ffff9c1f461ed080 ffff9c1f47c12000 rootfs rootfs /
Adding a redacted version of the previous comment:
#4 [ffffc099a8e97df0] unix_stream_connect at ffffffffbd0404ac 97df8: ffff9c1e761c4cf0 ffff9c20bed20d00 97e08: ffff9c1e46dfa400 ffff9c1f8dec2d80 97e18: 7fffffffffffffff ffffc099a8e97e80 97e28: ffffffffbdb9d1c0 fffffff5a8e97e80 97e38: 0e666d1c6cf1cc00 ffff9c1e9b2e2d00 97e48: ffffc099a8e97e80 0000000000000000 97e58: 000055d5fc3f1c60 0000000000000000 97e68: 000000000000001d ffffffffbcf19c5a 97e78: 0000000000000002 732f6e75722f0001 97e88: 6a2f646d65747379 732f6c616e72756f 97e98: ffff0074756f6474 ffff9c1f461ed6a0 97ea8: ffffc099a8e97f58 00000000c000003e 97eb8: 0000000000000000 ffffffffbc803d23 97ec8: ffff9c20c5a47698 ffff9c20c5a47698 97ed8: ffffffffbc983b99 0000000000000080 97ee8: ffffc099a8e97f58 ffffc099a8e97f58 97ef8: 0000000000000000 0e666d1c6cf1cc00 97f08: 000000000000002a ffffc099a8e97f58 97f18: 0000000000000000 0000000000000000 97f28: 0000000000000000 ffffffffbcf19ca6 97f38: ffffffffbc80419b 97f40: 0000000000000000 0000000000000000 97f50: ffffffffbd2000ad 64_after_ hwframe at ffffffffbd2000ad
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
#5 [ffffc099a8e97e70] __sys_connect at ffffffffbcf19c5a
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
ffffc099a8e
#6 [ffffc099a8e97f30] __x64_sys_connect at ffffffffbcf19ca6
ffffc099a8e
#7 [ffffc099a8e97f38] do_syscall_64 at ffffffffbc80419b
ffffc099a8e
ffffc099a8e
#8 [ffffc099a8e97f50] entry_SYSCALL_
RIP: 00007f8fdc558aa7 RSP: 00007ffdd43c8a20 RFLAGS: 00000293
RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007f8fdc558aa7
RDX: 000000000000001d RSI: 000055d5fc3f1c60 RDI: 0000000000000003
RBP: 000055d5fc3f1c60 R8: 0000000000000000 R9: 00007ffdd43c8ff4
R10: 00007ffdd43c8c98 R11: 0000000000000293 R12: 000000000000001d
R13: 0000000000000001 R14: 0000000000000000 R15: 0000000000000003
ORIG_RAX: 000000000000002a CS: 0033 SS: 002b
At frame4 (unix_stream_ connect) we poked at some random 0xffff addresses until we found the right one: sa_family_ t sun_family;
crash> ptype struct sockaddr_un
type = struct sockaddr_un {
__kernel_
char sun_path[108];
}
crash> p *(struct sockaddr_un *) 0xffffc099a8e97e80 journal/ stdout\ 000\377\ 377\240\ 326\036F\ 037\234\ 377\377X\ 177騙\300\ 377\377> \000\000\ 300\000\ 000\000\ 000\000\ 000\000\ 000\000\ 000\000\ 000#=\200\ 274\377\ 377\377\ 377\230v\ 244\305 \234\377\ 377\230v\ 244\305 \234 231;\230\ 274\377\ 377\377\ 377\200\ 000\000\ 000\000\ 000\000\ 000X\177騙\ 300"
$8 = {
sun_family = 1,
sun_path = "/run/systemd/
\377\377\
}
So all these (podman) processes are trying to talk to /run/systemd/ journal/ stdout
Which, according to https:/ /unix.stackexch ange.com/ questions/ 205883/ understand- logging- in-linux/ 294206# 294206 is: journal/ stdout for log data coming from systemd-managed services."
"It listens on the AF_LOCAL stream socket at /run/systemd/
So we started looking at the filesystem usr/lib/ debug/lib/ modules/ 4.18.0- 240.10. 1.el8_3. x86_64/ kernel/ fs/xfs/ xfs.ko. debug
crash> mod -s xfs
MODULE NAME SIZE OBJECT FILE
ffffffffc0652600 xfs 1511424 /tmp/kernel/
crash> mount
MOUNT SUPERBLK TYPE DEVNAME DIRNAME
ffff9c1f461ed080 ffff9c1f47c12000 rootfs rootfs /
crash> struct xfs_sb ffff9c222d5ca000 21408, 377\377\ 377\377\ 377\177\ 200'd\300\ 377\377\ 377\377" 19744, 23296, 15712, 377\377\ 000\000\ 000\000\ 000\000\ 000", 07192, 62113, 31200, 74496,
struct xfs_sb {
sb_magicnum = 780673024,
sb_blocksize = 4294941730,
sb_dblocks = 184466342693367
sb_rblocks = 51803848705,
sb_rextents = 4096,
sb_uuid = {
b = "\377\377\
},
sb_logstart = 184467440726419
sb_rootino = 0,
sb_rbmino = 184467440726419
sb_rsumino = 184467440726419
sb_rextsize = 1879113728,
sb_agblocks = 0,
sb_agcount = 1,
sb_rbmblocks = 0,
sb_logblocks = 1481003842,
sb_versionnum = 0,
sb_sectsize = 0,
sb_inodesize = 43264,
sb_inopblock = 25688,
sb_fname = "\"\234\
sb_blocklog = 120 'x',
sb_sectlog = 160 '\240',
sb_inodelog = 92 '\\',
sb_inopblog = 45 '-',
sb_agblklog = 34 '"',
sb_rextslog = 156 '\234',
sb_inprogress = 255 '\377',
sb_imax_pct = 255 '\377',
sb_icount = 184466342693367
sb_ifree = 0,
sb_fdblocks = 184466342702623
sb_frextents = 974957576193,
sb_uquotino = 184466342702629
sb_gquotino = 184467440726423
sb_qflags = 0,
sb_flags = 0 '\000',
....
The key here is likely: 07192,
sb_icount = 184466342693367
sb_ifree = 0,
I.e. no inodes are free? The inode count seems quite frankly impossibly large, so maybe we need to understand things a bit better here.