Aight, thanks Seth for the execsnoop link, excellent stuff to have in your tool-box. I just re-ran my test-case, and this is the new strace output, with the correlating execsnoop snippet.
select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "\0\0\0\1\v", 1024) = 5 select(5, [3 4], [4], NULL, NULL) = 1 (out [4]) write(4, "\0\0\1<\f\0\0\0\1\0\0\1\27\0\0\0\7ssh-rsa\0\0\0\3\1\0\1\0"..., 320) = 320 select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "\0\0\2\231\r\0\0\1\27\0\0\0\7ssh-rsa\0\0\0\3\1\0\1\0\0\1\1\0"..., 1024) = 669 select(5, [3 4], [4], NULL, NULL) = 1 (out [4]) write(4, "\0\0\1\24\16\0\0\1\17\0\0\0\7ssh-rsa\0\0\1\0z4DFM\312_\377"..., 280) = 280 select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "", 1024) = 0 close(4) = 0 select(4, [3], [], NULL, NULL) = 1 (in [3]) accept(3, {sa_family=AF_LOCAL, NULL}, [2]) = 4 getsockopt(4, SOL_SOCKET, SO_PEERCRED, {pid=24507, uid=1000, gid=1000}, [12]) = 0 getuid() = 1000 fcntl(4, F_GETFL) = 0x2 (flags O_RDWR) fcntl(4, F_SETFL, O_RDWR|O_NONBLOCK) = 0 select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "\0\0\0\1\v", 1024) = 5 select(5, [3 4], [4], NULL, NULL) = 1 (out [4]) write(4, "\0\0\1<\f\0\0\0\1\0\0\1\27\0\0\0\7ssh-rsa\0\0\0\3\1\0\1\0"..., 320) = 320 select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "", 1024) = 0 close(4) = 0 select(4, [3], [], NULL, NULL) = 1 (in [3]) accept(3, {sa_family=AF_LOCAL, NULL}, [2]) = 4 getsockopt(4, SOL_SOCKET, SO_PEERCRED, {pid=24900, uid=1000, gid=1000}, [12]) = 0 getuid() = 1000 fcntl(4, F_GETFL) = 0x2 (flags O_RDWR) fcntl(4, F_SETFL, O_RDWR|O_NONBLOCK) = 0 select(5, [3 4], [], NULL, NULL) = 1 (in [4]) read(4, "", 1024) = 0 close(4) = 0 select(4, [3], [], NULL, NULL) = ? ERESTARTNOHAND (To be restarted if no handler) --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=24900, si_uid=0} --- unlink("/tmp/ssh-vj0FsYTMJ3y0/agent.23119") = 0 rmdir("/tmp/ssh-vj0FsYTMJ3y0") = 0 close(-1) = -1 EBADF (Bad file descriptor) exit_group(2) = ? +++ exited with 2 +++
./execsnoop Tracing exec()s. Ctrl-C to end. Instrumenting sys_execve PID PPID ARGS 24896 24894 cat -v trace_pipe 24895 24891 gawk -v o=1 -v opt_name=0 -v name= -v opt_duration=0 [...] 24897 24515 ssh playpen 24898 24897 sh [?] 24899 24898 /usr/bin/xauth list :0.0 24900 1326 /usr/sbin/sshd -D -R 24903 24902 /sbin/pam-tmpdir-helper 24904 24902 /bin/sh -c command -v debian-sa1 > /dev/null && debian-sa1 1 1 24905 24904 debian-sa1 1 1 24907 24900 /sbin/pam-tmpdir-helper 24908 1770 /usr/lib/ConsoleKit/ck-collect-session-info --uid 0 --pid 24900 24909 1770 /usr/lib/ConsoleKit/run-session.d/pam-foreground-compat.ck 24910 24900 sh [?] 24911 24910 /usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run-parts --lsbsysinit /etc/update-motd.d 24912 24911 /etc/update-motd.d/00-header 24913 24912 uname -o 24914 24912 uname -r 24915 24912 uname -m 24916 24911 /etc/update-motd.d/10-help-text 24917 24916 uname -r 24918 24916 grep -qs \-server 24919 24911 /etc/update-motd.d/90-updates-available 24920 24919 apt-config shell StateDir Dir::State 24921 24920 /usr/bin/dpkg 24922 24919 apt-config shell ListDir Dir::State::Lists 24923 24922 /usr/bin/dpkg 24924 24919 apt-config shell EtcDir Dir::Etc 24925 24924 /usr/bin/dpkg 24926 24919 apt-config shell SourceList Dir::Etc::sourcelist 24927 24926 /usr/bin/dpkg 24928 24919 find /var/lib/apt//lists/ /etc/apt//sources.list -type f -newer /var/lib/update-notifier/updates-available -print -quit [...] 24929 24919 cat /var/lib/update-notifier/updates-available 24930 24911 /etc/update-motd.d/91-release-upgrade 24932 24931 lsb_release -sd 24933 24931 cut -d -f4 24930 0 /usr/lib/ubuntu-release-upgrader/release-upgrade-motd 24934 24930 date +%s 24935 24930 stat -c %Y /var/lib/ubuntu-release-upgrader/release-upgrade-available 24936 24930 expr 1429662656 + 86400 24937 24911 /etc/update-motd.d/98-fsck-at-reboot 24938 24937 stat -c %Y /var/lib/update-notifier/fsck-at-reboot 24940 24939 awk {print $1} /proc/uptime 24939 24937 date -d now - 709483.55 seconds +%s 24941 24937 date +%s 24943 24942 mount 24944 24942 awk $5 ~ /^ext(2|3|4)$/ { print $1 }
Looks like the culprit is sshd?
Aight, thanks Seth for the execsnoop link, excellent stuff to have in your tool-box. I just re-ran my test-case, and this is the new strace output, with the correlating execsnoop snippet.
select(5, [3 4], [], NULL, NULL) = 1 (in [4]) \f\0\0\ 0\1\0\0\ 1\27\0\ 0\0\7ssh- rsa\0\0\ 0\3\1\0\ 1\0"... , 320) = 320 231\r\0\ 0\1\27\ 0\0\0\7ssh- rsa\0\0\ 0\3\1\0\ 1\0\0\1\ 1\0"... , 1024) = 669 24\16\0\ 0\1\17\ 0\0\0\7ssh- rsa\0\0\ 1\0z4DFM\ 312_\377" ..., 280) = 280 AF_LOCAL, NULL}, [2]) = 4 \f\0\0\ 0\1\0\0\ 1\27\0\ 0\0\7ssh- rsa\0\0\ 0\3\1\0\ 1\0"... , 320) = 320 AF_LOCAL, NULL}, [2]) = 4 "/tmp/ssh- vj0FsYTMJ3y0/ agent.23119" ) = 0 /tmp/ssh- vj0FsYTMJ3y0" ) = 0
read(4, "\0\0\0\1\v", 1024) = 5
select(5, [3 4], [4], NULL, NULL) = 1 (out [4])
write(4, "\0\0\1<
select(5, [3 4], [], NULL, NULL) = 1 (in [4])
read(4, "\0\0\2\
select(5, [3 4], [4], NULL, NULL) = 1 (out [4])
write(4, "\0\0\1\
select(5, [3 4], [], NULL, NULL) = 1 (in [4])
read(4, "", 1024) = 0
close(4) = 0
select(4, [3], [], NULL, NULL) = 1 (in [3])
accept(3, {sa_family=
getsockopt(4, SOL_SOCKET, SO_PEERCRED, {pid=24507, uid=1000, gid=1000}, [12]) = 0
getuid() = 1000
fcntl(4, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(4, F_SETFL, O_RDWR|O_NONBLOCK) = 0
select(5, [3 4], [], NULL, NULL) = 1 (in [4])
read(4, "\0\0\0\1\v", 1024) = 5
select(5, [3 4], [4], NULL, NULL) = 1 (out [4])
write(4, "\0\0\1<
select(5, [3 4], [], NULL, NULL) = 1 (in [4])
read(4, "", 1024) = 0
close(4) = 0
select(4, [3], [], NULL, NULL) = 1 (in [3])
accept(3, {sa_family=
getsockopt(4, SOL_SOCKET, SO_PEERCRED, {pid=24900, uid=1000, gid=1000}, [12]) = 0
getuid() = 1000
fcntl(4, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(4, F_SETFL, O_RDWR|O_NONBLOCK) = 0
select(5, [3 4], [], NULL, NULL) = 1 (in [4])
read(4, "", 1024) = 0
close(4) = 0
select(4, [3], [], NULL, NULL) = ? ERESTARTNOHAND (To be restarted if no handler)
--- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=24900, si_uid=0} ---
unlink(
rmdir("
close(-1) = -1 EBADF (Bad file descriptor)
exit_group(2) = ?
+++ exited with 2 +++
./execsnoop tmpdir- helper tmpdir- helper ConsoleKit/ ck-collect- session- info --uid 0 --pid 24900 ConsoleKit/ run-session. d/pam-foregroun d-compat. ck local/sbin: /usr/local/ bin:/usr/ sbin:/usr/ bin:/sbin: /bin run-parts --lsbsysinit /etc/update-motd.d motd.d/ 00-header motd.d/ 10-help- text motd.d/ 90-updates- available :sourcelist apt//lists/ /etc/apt/ /sources. list -type f -newer /var/lib/ update- notifier/ updates- available -print -quit [...] update- notifier/ updates- available motd.d/ 91-release- upgrade ubuntu- release- upgrader/ release- upgrade- motd ubuntu- release- upgrader/ release- upgrade- available motd.d/ 98-fsck- at-reboot update- notifier/ fsck-at- reboot
Tracing exec()s. Ctrl-C to end.
Instrumenting sys_execve
PID PPID ARGS
24896 24894 cat -v trace_pipe
24895 24891 gawk -v o=1 -v opt_name=0 -v name= -v opt_duration=0 [...]
24897 24515 ssh playpen
24898 24897 sh [?]
24899 24898 /usr/bin/xauth list :0.0
24900 1326 /usr/sbin/sshd -D -R
24903 24902 /sbin/pam-
24904 24902 /bin/sh -c command -v debian-sa1 > /dev/null && debian-sa1 1 1
24905 24904 debian-sa1 1 1
24907 24900 /sbin/pam-
24908 1770 /usr/lib/
24909 1770 /usr/lib/
24910 24900 sh [?]
24911 24910 /usr/bin/env -i PATH=/usr/
24912 24911 /etc/update-
24913 24912 uname -o
24914 24912 uname -r
24915 24912 uname -m
24916 24911 /etc/update-
24917 24916 uname -r
24918 24916 grep -qs \-server
24919 24911 /etc/update-
24920 24919 apt-config shell StateDir Dir::State
24921 24920 /usr/bin/dpkg
24922 24919 apt-config shell ListDir Dir::State::Lists
24923 24922 /usr/bin/dpkg
24924 24919 apt-config shell EtcDir Dir::Etc
24925 24924 /usr/bin/dpkg
24926 24919 apt-config shell SourceList Dir::Etc:
24927 24926 /usr/bin/dpkg
24928 24919 find /var/lib/
24929 24919 cat /var/lib/
24930 24911 /etc/update-
24932 24931 lsb_release -sd
24933 24931 cut -d -f4
24930 0 /usr/lib/
24934 24930 date +%s
24935 24930 stat -c %Y /var/lib/
24936 24930 expr 1429662656 + 86400
24937 24911 /etc/update-
24938 24937 stat -c %Y /var/lib/
24940 24939 awk {print $1} /proc/uptime
24939 24937 date -d now - 709483.55 seconds +%s
24941 24937 date +%s
24943 24942 mount
24944 24942 awk $5 ~ /^ext(2|3|4)$/ { print $1 }
Looks like the culprit is sshd?