Comment 5 for bug 1923607

Revision history for this message
Michele Baldessari (michele) wrote :

Adding a redacted version of the previous comment:

#4 [ffffc099a8e97df0] unix_stream_connect at ffffffffbd0404ac
    ffffc099a8e97df8: ffff9c1e761c4cf0 ffff9c20bed20d00
    ffffc099a8e97e08: ffff9c1e46dfa400 ffff9c1f8dec2d80
    ffffc099a8e97e18: 7fffffffffffffff ffffc099a8e97e80
    ffffc099a8e97e28: ffffffffbdb9d1c0 fffffff5a8e97e80
    ffffc099a8e97e38: 0e666d1c6cf1cc00 ffff9c1e9b2e2d00
    ffffc099a8e97e48: ffffc099a8e97e80 0000000000000000
    ffffc099a8e97e58: 000055d5fc3f1c60 0000000000000000
    ffffc099a8e97e68: 000000000000001d ffffffffbcf19c5a
 #5 [ffffc099a8e97e70] __sys_connect at ffffffffbcf19c5a
    ffffc099a8e97e78: 0000000000000002 732f6e75722f0001
    ffffc099a8e97e88: 6a2f646d65747379 732f6c616e72756f
    ffffc099a8e97e98: ffff0074756f6474 ffff9c1f461ed6a0
    ffffc099a8e97ea8: ffffc099a8e97f58 00000000c000003e
    ffffc099a8e97eb8: 0000000000000000 ffffffffbc803d23
    ffffc099a8e97ec8: ffff9c20c5a47698 ffff9c20c5a47698
    ffffc099a8e97ed8: ffffffffbc983b99 0000000000000080
    ffffc099a8e97ee8: ffffc099a8e97f58 ffffc099a8e97f58
    ffffc099a8e97ef8: 0000000000000000 0e666d1c6cf1cc00
    ffffc099a8e97f08: 000000000000002a ffffc099a8e97f58
    ffffc099a8e97f18: 0000000000000000 0000000000000000
    ffffc099a8e97f28: 0000000000000000 ffffffffbcf19ca6
 #6 [ffffc099a8e97f30] __x64_sys_connect at ffffffffbcf19ca6
    ffffc099a8e97f38: ffffffffbc80419b
 #7 [ffffc099a8e97f38] do_syscall_64 at ffffffffbc80419b
    ffffc099a8e97f40: 0000000000000000 0000000000000000
    ffffc099a8e97f50: ffffffffbd2000ad
 #8 [ffffc099a8e97f50] entry_SYSCALL_64_after_hwframe at ffffffffbd2000ad
    RIP: 00007f8fdc558aa7 RSP: 00007ffdd43c8a20 RFLAGS: 00000293
    RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007f8fdc558aa7
    RDX: 000000000000001d RSI: 000055d5fc3f1c60 RDI: 0000000000000003
    RBP: 000055d5fc3f1c60 R8: 0000000000000000 R9: 00007ffdd43c8ff4
    R10: 00007ffdd43c8c98 R11: 0000000000000293 R12: 000000000000001d
    R13: 0000000000000001 R14: 0000000000000000 R15: 0000000000000003
    ORIG_RAX: 000000000000002a CS: 0033 SS: 002b

At frame4 (unix_stream_connect) we poked at some random 0xffff addresses until we found the right one:
crash> ptype struct sockaddr_un
type = struct sockaddr_un {
    __kernel_sa_family_t sun_family;
    char sun_path[108];
}

crash> p *(struct sockaddr_un *) 0xffffc099a8e97e80
$8 = {
  sun_family = 1,
  sun_path = "/run/systemd/journal/stdout\000\377\377\240\326\036F\037\234\377\377X\177騙\300\377\377>\000\000\300\000\000\000\000\000\000\000\000\000\000\000\000#=\200\274\377\377\377\377\230v\244\305 \234\377\377\230v\244\305 \234
\377\377\231;\230\274\377\377\377\377\200\000\000\000\000\000\000\000X\177騙\300"
}

So all these (podman) processes are trying to talk to /run/systemd/journal/stdout

Which, according to https://unix.stackexchange.com/questions/205883/understand-logging-in-linux/294206#294206 is:
"It listens on the AF_LOCAL stream socket at /run/systemd/journal/stdout for log data coming from systemd-managed services."

So we started looking at the filesystem
crash> mod -s xfs
     MODULE NAME SIZE OBJECT FILE
ffffffffc0652600 xfs 1511424 /tmp/kernel/usr/lib/debug/lib/modules/4.18.0-240.10.1.el8_3.x86_64/kernel/fs/xfs/xfs.ko.debug

crash> mount
     MOUNT SUPERBLK TYPE DEVNAME DIRNAME
ffff9c1f461ed080 ffff9c1f47c12000 rootfs rootfs /

crash> struct xfs_sb ffff9c222d5ca000
struct xfs_sb {
  sb_magicnum = 780673024,
  sb_blocksize = 4294941730,
  sb_dblocks = 18446634269336721408,
  sb_rblocks = 51803848705,
  sb_rextents = 4096,
  sb_uuid = {
    b = "\377\377\377\377\377\377\377\177\200'd\300\377\377\377\377"
  },
  sb_logstart = 18446744072641919744,
  sb_rootino = 0,
  sb_rbmino = 18446744072641923296,
  sb_rsumino = 18446744072641915712,
  sb_rextsize = 1879113728,
  sb_agblocks = 0,
  sb_agcount = 1,
  sb_rbmblocks = 0,
  sb_logblocks = 1481003842,
  sb_versionnum = 0,
  sb_sectsize = 0,
  sb_inodesize = 43264,
  sb_inopblock = 25688,
  sb_fname = "\"\234\377\377\000\000\000\000\000\000\000",
  sb_blocklog = 120 'x',
  sb_sectlog = 160 '\240',
  sb_inodelog = 92 '\\',
  sb_inopblog = 45 '-',
  sb_agblklog = 34 '"',
  sb_rextslog = 156 '\234',
  sb_inprogress = 255 '\377',
  sb_imax_pct = 255 '\377',
  sb_icount = 18446634269336707192,
  sb_ifree = 0,
  sb_fdblocks = 18446634270262362113,
  sb_frextents = 974957576193,
  sb_uquotino = 18446634270262931200,
  sb_gquotino = 18446744072642374496,
  sb_qflags = 0,
  sb_flags = 0 '\000',
....

The key here is likely:
  sb_icount = 18446634269336707192,
  sb_ifree = 0,

I.e. no inodes are free? The inode count seems quite frankly impossibly large, so maybe we need to understand things a bit better here.