ok. so i have 2 other systems that are showing this failure now.
I was able to ssh into them, though.
walinux-agent had provisioned the user, populated ssh keys and then also started sshd (which it actually should not do).
it shouldn't start sshd because it is possibly doing that before sshd has the required facilities up (sshd starts on 'filesystem or runlevel [2345]).
that wouldn't seem to be the problem here, and actually has allowed us into the instance to debug.
ok. so i have 2 other systems that are showing this failure now.
I was able to ssh into them, though.
walinux-agent had provisioned the user, populated ssh keys and then also started sshd (which it actually should not do).
it shouldn't start sshd because it is possibly doing that before sshd has the required facilities up (sshd starts on 'filesystem or runlevel [2345]).
that wouldn't seem to be the problem here, and actually has allowed us into the instance to debug.
$ ls -tr --full-time /var/log/ upstart/ *.log upstart/ container- detect. log upstart/ console- setup.log upstart/ procps- virtual- filesystems. log upstart/ module- init-tools. log upstart/ procps- static- network- up.log upstart/ udev-fallback- graphics. log upstart/ ureadahead- other.log upstart/ rsyslog. log upstart/ dbus.log
-rw-r----- 1 root root 46 2013-06-28 15:07:55.843772000 +0000 /var/log/
-rw-r----- 1 root root 95 2013-06-28 15:07:56.095772000 +0000 /var/log/
-rw-r----- 1 root root 282 2013-06-28 15:07:56.183772000 +0000 /var/log/
-rw-r----- 1 root root 118 2013-06-28 15:07:56.311772000 +0000 /var/log/
-rw-r----- 1 root root 282 2013-06-28 15:07:58.310376600 +0000 /var/log/
-rw-r----- 1 root root 110 2013-06-28 15:08:02.993943800 +0000 /var/log/
-rw-r----- 1 root root 158 2013-06-28 15:08:09.876561300 +0000 /var/log/
-rw-r----- 1 root root 64 2013-06-28 15:09:30.346411301 +0000 /var/log/
-rw-r----- 1 root root 64 2013-06-28 15:09:30.370411301 +0000 /var/log/
$ cat /proc/mounts nodev,noexec, relatime 0 0 nodev,noexec, relatime 0 0 size=335336k, nr_inodes= 83834,mode= 755 0 0 noexec, relatime, gid=5,mode= 620,ptmxmode= 000 0 0 relatime, size=137672k, mode=755 0 0 by-uuid/ 65a0705a- 7afe-482f- 917d-c59e75cf0c 52 / ext4 rw,relatime, user_xattr, barrier= 1,data= ordered, discard 0 0 fuse/connection s fusectl rw,relatime 0 0 security securityfs rw,relatime 0 0 nodev,noexec, relatime, size=5120k 0 0 nodev,relatime 0 0 user_xattr, barrier= 1,data= ordered 0 0
rootfs / rootfs rw 0 0
sysfs /sys sysfs rw,nosuid,
proc /proc proc rw,nosuid,
udev /dev devtmpfs rw,relatime,
devpts /dev/pts devpts rw,nosuid,
tmpfs /run tmpfs rw,nosuid,
/dev/disk/
none /sys/fs/
none /sys/kernel/debug debugfs rw,relatime 0 0
none /sys/kernel/
none /run/lock tmpfs rw,nosuid,
none /run/shm tmpfs rw,nosuid,
/dev/sdb1 /mnt/resource ext4 rw,relatime,
$ cat /etc/fstab 7afe-482f- 917d-c59e75cf0c 52 / ext4 defaults,discard 0 0
UUID=65a0705a-
mountall is not running.
$ sudo status mountall
mountall stop/waiting
$ ls -altr /var/run/landscape
ls: cannot access /var/run/landscape: No such file or directory
$ runlevel
N 2
$ ps axw socket- bridge --daemon if-up.d/ ntpdate ntpdate- ifup hv_kvp_ daemon_ 3.2.0-48- virtual
..
root 389 1 0 15:07 ? 00:00:00 upstart-udev-bridge --daemon
root 391 1 0 15:07 ? 00:00:00 /sbin/udevd --daemon
root 508 1 0 15:07 ? 00:00:00 /usr/bin/python /usr/sbin/waagent -daemo
root 574 391 0 15:07 ? 00:00:00 /sbin/udevd --daemon
root 577 391 0 15:07 ? 00:00:00 /sbin/udevd --daemon
root 598 2 0 15:07 ? 00:00:00 [kpsmoused]
root 633 1 0 15:07 ? 00:00:00 upstart-
root 906 2 0 15:08 ? 00:00:00 [jbd2/sdb1-8]
root 907 2 0 15:08 ? 00:00:00 [ext4-dio-unwrit]
root 931 508 0 15:08 ? 00:00:00 [sh] <defunct>
root 1015 1 0 15:08 ? 00:00:00 dhclient3 -e IF_METRIC=100 -pf /var/run/
root 1025 1 0 15:08 ? 00:00:00 /bin/sh /etc/network/
root 1028 1025 0 15:08 ? 00:00:00 lockfile-create /var/lock/
root 1121 1 0 15:09 ? 00:00:00 /usr/sbin/sshd -D
syslog 1137 1 0 15:09 ? 00:00:00 rsyslogd -c5
102 1142 1 0 15:09 ? 00:00:00 dbus-daemon --system --fork --activation
root 1200 1 0 15:09 tty4 00:00:00 /sbin/getty -8 38400 tty4
root 1207 1 0 15:09 tty5 00:00:00 /sbin/getty -8 38400 tty5
root 1214 1 0 15:09 tty2 00:00:00 /sbin/getty -8 38400 tty2
root 1215 1 0 15:09 tty3 00:00:00 /sbin/getty -8 38400 tty3
root 1218 1 0 15:09 tty6 00:00:00 /sbin/getty -8 38400 tty6
root 1248 1 0 15:09 ? 00:00:00 /usr/sbin/
root 1250 1 0 15:09 ? 00:00:00 acpid -c /etc/acpi/events -s /var/run/ac
root 1251 1 0 15:09 ? 00:00:00 cron
daemon 1252 1 0 15:09 ? 00:00:00 atd
root 1265 1 0 15:09 tty1 00:00:00 /sbin/getty -8 38400 tty1
whoopsie 1279 1 0 15:09 ? 00:00:00 whoopsie
root 1308 1121 0 15:23 ? 00:00:00 sshd: test [priv]
test 1412 1308 0 15:24 ? 00:00:00 sshd: test@pts/0
test 1413 1412 0 15:24 pts/0 00:00:01 -bash
root 1755 2 0 15:33 ? 00:00:00 [kworker/0:0]
root 2054 2 0 15:38 ? 00:00:00 [kworker/0:2]
root 2274 2 0 15:43 ? 00:00:00 [kworker/0:1]
test 2450 1413 0 15:45 pts/0 00:00:00 ps -ef
Note, it seems that 'lockfile-create /var/lock/ ntpdate- ifup' is hung.
$ sudo sh -c "tr '\0' ' ' < /proc/1025/environ" ; echo local/sbin: /usr/local/ bin:/usr/ sbin:/usr/ bin:/sbin: /bin IF_METRIC=100 IFACE=eth0 PWD=/var/ lib/waagent
METHOD=dhcp MODE=start LOGICAL=eth0 PHASE=post-up ADDRFAM=inet VERBOSITY=0 PATH=/usr/
$ ls -l /proc/1025/cmdline --full-time
-r--r--r-- 1 root root 0 2013-06-28 15:23:53.965119600 +0000 /proc/1025/cmdline
$ ls -l /run/network/ --full-time network- up-emitted
total 4
-rw-r--r-- 1 root root 16 2013-06-28 15:08:13.333041000 +0000 ifstate
-rw-r--r-- 1 root root 0 2013-06-28 15:08:13.321041000 +0000 ifup.eth0
-rw-r--r-- 1 root root 0 2013-06-28 15:07:57.774376600 +0000 ifup.lo
drwxr-xr-x 2 root root 40 2013-06-28 15:07:58.298376600 +0000 static-