Script started on Tue 20 Dec 2016 09:31:29 AM CST root@10:~# virsh console dotg9 Connected to domain dotg9 Escape character is ^] root@dotg9:~# df -h Filesystem Size Used Avail Use% Mounted on udev 2.7G 0 2.7G 0% /dev tmpfs 561M 16M 545M 3% /run /dev/mapper/mpatha-part2 19G 3.5G 15G 20% / tmpfs 2.8G 0 2.8G 0% /dev/shm tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 2.8G 0 2.8G 0% /sys/fs/cgroup tmpfs 561M 0 561M 0% /run/user/0 root@dotg9:~# multipath -l mpatha (36005076802818bda300000000000049f) dm-0 IBM,2145 size=20G features='1 queue_if_no_path' hwhandler='0' wp=rw |-+- policy='round-robin 0' prio=0 status=active | `- 1:0:0:0 sdb 8:16 active undef running `-+- policy='round-robin 0' prio=0 status=enabled `- 0:0:0:1 sda 8:0 active undef running root@dotg9:~# echo c > /proc/sysrq-trigger [ 117.057394] sysrq: SysRq : Trigger a crash [ 117.057586] Unable to handle kernel paging request for data at address 0x00000000 [ 117.057668] Faulting instruction address: 0xc0000000006ccb18 [ 117.057778] Oops: Kernel access of bad area, sig: 11 [#1] [ 117.057833] SMP NR_CPUS=2048 NUMA pSeries [ 117.057896] Modules linked in: vmx_crypto sunrpc autofs4 dm_round_robin crc32c_vpmsum virtio_scsi virtio_net scsi_dh_emc scsi_dh_rdac scsi_dh_alua dm_multipath [ 117.058116] CPU: 3 PID: 1741 Comm: bash Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 117.058198] task: c000000003a84200 task.stack: c000000003ae0000 [ 117.058265] NIP: c0000000006ccb18 LR: c0000000006cdc3c CTR: c0000000006ccae0 [ 117.058345] REGS: c000000003ae39d0 TRAP: 0300 Not tainted (4.8.0-32-generic) [ 117.058425] MSR: 8000000000009033 CR: 28242222 XER: 20000000 [ 117.058620] CFAR: c00000000006ba6c DAR: 0000000000000000 DSISR: 42000000 SOFTE: 1 GPR00: c0000000006cdc3c c000000003ae3c50 c0000000014a6600 0000000000000063 GPR04: c00000017fecaca0 c00000017fedfb40 0000000000000114 0000000000004f38 GPR08: 0000000000000007 0000000000000001 0000000000000000 0000000000000001 GPR12: c0000000006ccae0 c000000007b81b00 ffffffffffffffff 0000000022000000 GPR16: 0000000010170dd0 0000010013530258 00000000101405c8 00000000100c70a0 GPR20: 0000000000000000 000000001017dd78 0000000010140460 0000000000000000 GPR24: 0000000010153320 000000001017b628 c0000000013be7a0 0000000000000004 GPR28: c0000000013beb60 0000000000000063 c000000001372590 0000000000000000 [ 117.059743] NIP [c0000000006ccb18] sysrq_handle_crash+0x38/0x50 [ 117.059812] LR [c0000000006cdc3c] __handle_sysrq+0xec/0x280 [ 117.059866] Call Trace: [ 117.059907] [c000000003ae3c50] [c000000000b52f58] printk+0x5c/0x70 (unreliable) [ 117.060004] [c000000003ae3c70] [c0000000006cdc3c] __handle_sysrq+0xec/0x280 [ 117.060086] [c000000003ae3d10] [c0000000006ce3e0] write_sysrq_trigger+0x70/0x90 [ 117.060191] [c000000003ae3d40] [c0000000003c499c] proc_reg_write+0x8c/0xd0 [ 117.060275] [c000000003ae3d70] [c000000000322ae0] __vfs_write+0x40/0x80 [ 117.060357] [c000000003ae3d90] [c000000000323e84] vfs_write+0xd4/0x270 [ 117.060440] [c000000003ae3de0] [c000000000325a8c] SyS_write+0x6c/0x110 [ 117.060522] [c000000003ae3e30] [c000000000009584] system_call+0x38/0xec [ 117.060601] Instruction dump: [ 117.060643] 7c0802a6 f8010010 60000000 7c0802a6 f8010010 f821ffe1 3d22001a 3949c9e0 [ 117.060783] 39200001 912a0000 7c0004ac 39400000 <992a0000> 38210020 e8010010 7c0803a6 [ 117.060966] ---[ end trace 321e2498584b7dce ]--- [ 117.063803] [ 117.063846] Sending IPI to other CPUs [ 117.064900] IPI complete I'm in purgatory [ 0.000000] Allocated 65536 bytes for 8 pacas at c00000000fff0000 [ 0.000000] Adjusted logical boot cpu id: logical 0 physical 24 [ 0.000000] Page sizes from device-tree: [ 0.000000] base_shift=12: shift=12, sllp=0x0000, avpnm=0x00000000, tlbiel=1, penc=0 [ 0.000000] base_shift=16: shift=16, sllp=0x0110, avpnm=0x00000000, tlbiel=1, penc=1 [ 0.000000] Page orders: linear mapping = 16, virtual = 16, io = 16, vmemmap = 16 [ 0.000000] Using 1TB segments [ 0.000000] Initializing hash mmu with SLB [ 0.000000] Linux version 4.8.0-32-generic (buildd@bos01-ppc64el-001) (gcc version 5.4.0 20160609 (Ubuntu/IBM 5.4.0-6ubuntu1~16.04.4) ) #34~16.04.1-Ubuntu SMP Tue Dec 13 17:01:57 UTC 2016 (Ubuntu 4.8.0-32.34~16.04.1-generic 4.8.11) [ 0.000000] Found initrd at 0xc000000009810000:0xc00000000a6b7272 [ 0.000000] Using pSeries machine description [ 0.000000] bootconsole [udbg0] enabled [ 0.000000] Partition configured for 4 cpus, operating system maximum is 1. [ 0.000000] CPU maps initialized for 1 thread per core [ 0.000000] (thread shift is 0) -> smp_release_cpus() spinning_secondaries = 3 <- smp_release_cpus() [ 0.000000] ----------------------------------------------------- [ 0.000000] ppc64_pft_size = 0x1a [ 0.000000] phys_mem_size = 0x28010000 [ 0.000000] dcache_bsize = 0x80 [ 0.000000] icache_bsize = 0x80 [ 0.000000] cpu_features = 0x37fc7a6c18500249 [ 0.000000] possible = 0x3fffffff18500649 [ 0.000000] always = 0x0000000018100040 [ 0.000000] cpu_user_features = 0xdc0065c2 0xef000000 [ 0.000000] mmu_features = 0x78000001 [ 0.000000] firmware_features = 0x00000001405a440b [ 0.000000] htab_hash_mask = 0x7ffff [ 0.000000] physical_start = 0x8000000 [ 0.000000] ----------------------------------------------------- [ 0.000000] Node 0 Memory: 0x0-0x30000000 [ 0.000000] numa: Initmem setup node 0 [mem 0x00000000-0x2fffffff] [ 0.000000] numa: NODE_DATA [mem 0x27fce300-0x27fd7fff] [ 0.000000] PCI host bridge /pci@800000020000000 ranges: [ 0.000000] IO 0x0000010080000000..0x000001008000ffff -> 0x0000000000000000 [ 0.000000] MEM 0x00000100a0000000..0x000001101fffffff -> 0x0000000080000000 [ 0.000000] PPC64 nvram contains 65536 bytes [ 0.000000] Top of RAM: 0x30000000, Total RAM: 0x28010000 [ 0.000000] Memory hole size: 127MB [ 0.000000] Zone ranges: [ 0.000000] DMA [mem 0x0000000000000000-0x000000002fffffff] [ 0.000000] DMA32 empty [ 0.000000] Normal empty [ 0.000000] Movable zone start for each node [ 0.000000] Early memory node ranges [ 0.000000] node 0: [mem 0x0000000000000000-0x0000000027ffffff] [ 0.000000] node 0: [mem 0x000000002fff0000-0x000000002fffffff] [ 0.000000] Initmem setup node 0 [mem 0x0000000000000000-0x000000002fffffff] [ 0.000000] On node 0 totalpages: 10241 [ 0.000000] DMA zone: 11 pages used for memmap [ 0.000000] DMA zone: 0 pages reserved [ 0.000000] DMA zone: 10241 pages, LIFO batch:1 [ 0.000000] percpu: Embedded 3 pages/cpu @c000000027e00000 s148888 r0 d47720 u1048576 [ 0.000000] pcpu-alloc: s148888 r0 d47720 u1048576 alloc=1*1048576 [ 0.000000] pcpu-alloc: [0] 0 [ 0.000000] Built 1 zonelists in Node order, mobility grouping on. Total pages: 10230 [ 0.000000] Policy zone: DMA [ 0.000000] Kernel command line: BOOT_IMAGE=/boot/vmlinux-4.8.0-32-generic root=UUID=a557092e-9d7f-496e-a371-7a043e192ec9 ro loglevel=8 irqpoll nr_cpus=1 nousb systemd.unit=kdump-tools.service elfcorehdr=155584K [ 0.000000] Misrouted IRQ fixup and polling support enabled [ 0.000000] This may significantly impact system performance [ 0.000000] PID hash table entries: 4096 (order: -1, 32768 bytes) [ 0.000000] Sorting __ex_table... [ 0.000000] Memory: 417088K/655424K available (11648K kernel code, 1600K rwdata, 3500K rodata, 4608K init, 2961K bss, 238336K reserved, 0K cma-reserved) [ 0.000000] SLUB: HWalign=128, Order=0-3, MinObjects=0, CPUs=1, Nodes=1 [ 0.000000] Hierarchical RCU implementation. [ 0.000000] Build-time adjustment of leaf fanout to 64. [ 0.000000] RCU restricting CPUs from NR_CPUS=2048 to nr_cpu_ids=1. [ 0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=1 [ 0.000000] NR_IRQS:512 nr_irqs:512 16 [ 0.000000] pic: no ISA interrupt controller [ 0.000000] time_init: decrementer frequency = 512.000000 MHz [ 0.000000] time_init: processor frequency = 3425.000000 MHz [ 0.000001] clocksource: timebase: mask: 0xffffffffffffffff max_cycles: 0x761537d007, max_idle_ns: 440795202126 ns [ 0.000791] clocksource: timebase mult[1f40000] shift[24] registered [ 0.001276] clockevent: decrementer mult[83126e98] shift[32] cpu[0] [ 0.001787] Console: colour dummy device 80x25 [ 0.002135] console [hvc0] enabled [ 0.002135] console [hvc0] enabled [ 0.002415] bootconsole [udbg0] disabled [ 0.002415] bootconsole [udbg0] disabled [ 0.002776] pid_max: default: 32768 minimum: 301 [ 0.002900] Security Framework initialized [ 0.002923] Yama: becoming mindful. [ 0.002957] AppArmor: AppArmor initialized [ 0.003067] Dentry cache hash table entries: 131072 (order: 4, 1048576 bytes) [ 0.003333] Inode-cache hash table entries: 65536 (order: 3, 524288 bytes) [ 0.003480] Mount-cache hash table entries: 8192 (order: 0, 65536 bytes) [ 0.003523] Mountpoint-cache hash table entries: 8192 (order: 0, 65536 bytes) [ 0.004896] ftrace: allocating 30128 entries in 12 pages [ 0.016237] EEH: pSeries platform initialized [ 0.016300] POWER8 performance monitor hardware support registered [ 0.016341] power8-pmu: PMAO restore workaround active. [ 0.016949] Brought up 1 CPUs [ 0.016990] Node 0 CPUs: 0 [ 0.017176] devtmpfs: initialized [ 0.028136] evm: security.selinux [ 0.028167] evm: security.SMACK64 [ 0.028192] evm: security.SMACK64EXEC [ 0.028216] evm: security.SMACK64TRANSMUTE [ 0.028239] evm: security.SMACK64MMAP [ 0.028261] evm: security.ima [ 0.028282] evm: security.capability [ 0.028391] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns [ 0.028599] NET: Registered protocol family 16 [ 0.029368] EEH: No capable adapters found [ 0.029433] IBM eBus Device Driver [ 0.029561] cpuidle: using governor ladder [ 0.029584] cpuidle: using governor menu [ 0.029668] RTAS daemon started [ 0.029750] pstore: using zlib compression [ 0.029773] pstore: Registered nvram as persistent store backend Linux ppc64le #34~16.04.1-Ubun[ 0.030333] PCI: Probing PCI hardware [ 0.030384] PCI host bridge to bus 0000:00 [ 0.030408] pci_bus 0000:00: root bus resource [io 0x10000-0x1ffff] (bus address [0x0000-0xffff]) [ 0.030457] pci_bus 0000:00: root bus resource [mem 0x100a0000000-0x1101fffffff] (bus address [0x80000000-0xfffffffff]) [ 0.030513] pci_bus 0000:00: root bus resource [bus 00-ff] [ 0.061000] IOMMU table initialized, virtual merging enabled [ 0.061069] iommu: Adding device 0000:00:01.0 to group 0 [ 0.061149] iommu: Adding device 0000:00:02.0 to group 0 [ 0.061206] iommu: Adding device 0000:00:03.0 to group 0 [ 0.061265] iommu: Adding device 0000:00:04.0 to group 0 [ 0.061408] PCI: Probing PCI hardware done [ 0.062763] SCSI subsystem initialized [ 0.062808] libata version 3.00 loaded. [ 0.062856] vgaarb: loaded [ 0.062901] usbcore: registered new interface driver usbfs [ 0.062941] usbcore: registered new interface driver hub [ 0.062986] usbcore: registered new device driver usb [ 0.063030] pps_core: LinuxPPS API ver. 1 registered [ 0.063063] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti [ 0.063118] PTP clock support registered [ 0.063268] NetLabel: Initializing [ 0.063294] NetLabel: domain hash size = 128 [ 0.063324] NetLabel: protocols = UNLABELED CIPSOv4 [ 0.063366] NetLabel: unlabeled traffic allowed by default [ 0.063465] clocksource: Switched to clocksource timebase [ 0.070072] VFS: Disk quotas dquot_6.6.0 [ 0.070146] VFS: Dquot-cache hash table entries: 8192 (order 0, 65536 bytes) [ 0.070263] hugetlbfs: disabling because there are no supported hugepage sizes [ 0.070356] AppArmor: AppArmor Filesystem Enabled [ 0.071599] NET: Registered protocol family 2 [ 0.071754] TCP established hash table entries: 8192 (order: 0, 65536 bytes) [ 0.071819] TCP bind hash table entries: 8192 (order: 1, 131072 bytes) [ 0.071875] TCP: Hash tables configured (established 8192 bind 8192) [ 0.071921] UDP hash table entries: 2048 (order: 0, 65536 bytes) [ 0.071970] UDP-Lite hash table entries: 2048 (order: 0, 65536 bytes) [ 0.072040] NET: Registered protocol family 1 [ 0.131829] PCI: CLS 0 bytes, default 128 [ 0.131893] Trying to unpack rootfs image as initramfs... [ 0.331143] Freeing initrd memory: 14976K (c000000009810000 - c00000000a6b0000) [ 0.331653] futex hash table entries: 256 (order: -1, 32768 bytes) [ 0.331711] audit: initializing netlink subsys (disabled) [ 0.331754] audit: type=2000 audit(1482247924.324:1): initialized [ 0.331906] Initialise system trusted keyrings [ 0.331999] workingset: timestamp_bits=38 max_order=13 bucket_order=0 [ 0.333592] zbud: loaded [ 0.334185] squashfs: version 4.0 (2009/01/31) Phillip Lougher [ 0.334353] fuse init (API version 7.25) [ 0.334501] Allocating IMA blacklist keyring. [ 0.335339] Key type asymmetric registered [ 0.335364] Asymmetric key parser 'x509' registered [ 0.335425] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 246) [ 0.335509] io scheduler noop registered [ 0.335537] io scheduler deadline registered (default) [ 0.335574] io scheduler cfq registered [ 0.335668] pci_hotplug: PCI Hot Plug PCI Core version: 0.5 [ 0.335811] virtio-pci 0000:00:01.0: virtio_pci: leaving for legacy driver [ 0.351354] virtio-pci 0000:00:02.0: virtio_pci: leaving for legacy driver [ 0.379202] virtio-pci 0000:00:03.0: virtio_pci: leaving for legacy driver [ 0.411319] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled [ 0.412342] Linux agpgart interface v0.103 [ 0.414509] brd: module loaded [ 0.415331] loop: module loaded [ 0.415511] libphy: Fixed MDIO Bus: probed [ 0.415546] tun: Universal TUN/TAP device driver, 1.6 [ 0.415584] tun: (C) 1999-2004 Max Krasnyansky [ 0.415660] ibmveth: IBM Power Virtual Ethernet Driver 1.05 [ 0.415713] ehea: IBM eHEA ethernet device driver (Release EHEA_0107) [ 0.415775] PPP generic driver version 2.4.2 [ 0.415861] VFIO - User Level meta-driver version: 0.3 [ 0.417843] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver [ 0.417908] ehci-pci: EHCI PCI platform driver [ 0.417958] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver [ 0.418010] ohci-pci: OHCI PCI platform driver [ 0.418351] ohci-pci 0000:00:04.0: OHCI PCI host controller [ 0.418401] ohci-pci 0000:00:04.0: new USB bus registered, assigned bus number 1 [ 0.418565] ohci-pci 0000:00:04.0: irq 20, io mem 0x100e0000000 [ 0.475763] usb usb1: New USB device found, idVendor=1d6b, idProduct=0001 [ 0.475820] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1 [ 0.475880] usb usb1: Product: OHCI PCI host controller [ 0.475921] usb usb1: Manufacturer: Linux 4.8.0-32-generic ohci_hcd [ 0.475972] usb usb1: SerialNumber: 0000:00:04.0 [ 0.476093] hub 1-0:1.0: USB hub found [ 0.476150] hub 1-0:1.0: 3 ports detected [ 0.476335] uhci_hcd: USB Universal Host Controller Interface driver [ 0.476471] mousedev: PS/2 mouse device common for all mice [ 0.476600] rtc-generic rtc-generic: rtc core: registered rtc-generic as rtc0 [ 0.476672] i2c /dev entries driver [ 0.476770] device-mapper: uevent: version 1.0.3 [ 0.476862] device-mapper: ioctl: 4.35.0-ioctl (2016-06-23) initialised: dm-devel@redhat.com [ 0.476945] pseries_idle_driver registered [ 0.476985] ledtrig-cpu: registered to indicate activity on CPUs [ 0.477179] NET: Registered protocol family 10 [ 0.477395] NET: Registered protocol family 17 [ 0.477446] Key type dns_resolver registered [ 0.477612] registered taskstats version 1 [ 0.477652] Loading compiled-in X.509 certificates [ 0.479935] Loaded X.509 cert 'Build time autogenerated kernel key: beb24bb1779de8415f01800ee18c1ac8e810856b' [ 0.480147] zswap: loaded using pool lzo/zbud [ 0.499255] Key type big_key registered [ 0.500820] Key type trusted registered [ 0.502362] Key type encrypted registered [ 0.502404] AppArmor: AppArmor sha1 policy hashing enabled [ 0.502444] ima: No TPM chip found, activating TPM-bypass! [ 0.502500] evm: HMAC attrs: 0x1 [ 0.502680] rtc-generic rtc-generic: setting system clock to 2016-12-20 15:32:05 UTC (1482247925) [ 0.504258] Freeing unused kernel memory: 4608K (c000000008ee0000 - c000000009360000) [ 0.504358] This architecture does not have kernel memory protection. Loading, please wait... Begin: Loading multipath modules ... Success: loaded module dm-multipath. done. Begin: Loading multipath hardware handlers ... Failure: failed to load module scsi_dh_alua. Failure: failed to load module scsi_dh_rdac. Failure: failed to load module scsi_dh_emc. done. Begin: Starting multipathd ... done. [ 0.523937] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524236] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524300] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524500] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524654] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524788] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.524921] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.525053] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.525185] random: multipathd: uninitialized urandom read (16 bytes read) [ 0.525294] random: multipathd: uninitialized urandom read (16 bytes read) starting version 229 [ 0.571634] scsi host0: Virtio SCSI HBA [ 0.703728] scsi host1: Virtio SCSI HBA [ 22.003491] scsi 1:0:0:0: tag#0 abort [ 22.003838] scsi 0:0:0:0: tag#0 abort Begin: Loading essential drivers ... done. Begin: Running /scripts/init-premount ... done. Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done. Begin: Running /scripts/local-premount ... Begin: Waiting for udev to settle (multipath) ... done. [ 242.663489] INFO: task kworker/u2:0:6 blocked for more than 120 seconds. [ 242.663581] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 242.663628] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 242.663686] kworker/u2:0 D 0000000000000000 0 6 2 0x00000800 [ 242.663758] Workqueue: scsi_tmf_0 scmd_eh_abort_handler [ 242.663812] Call Trace: [ 242.663837] [c00000002607f6a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 242.663910] [c00000002607f870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 242.663972] [c00000002607f8d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 242.664032] [c00000002607f9b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 242.664091] [c00000002607f9e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 242.664159] [c00000002607fad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 242.664228] [c00000002607fb50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 242.664307] [c00000002607fbc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 242.664378] [c00000002607fc50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 242.664447] [c00000002607fce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 242.664507] [c00000002607fd80] [c0000000080f8224] kthread+0x114/0x140 [ 242.664571] [c00000002607fe30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c [ 242.664658] INFO: task systemd-udevd:173 blocked for more than 120 seconds. [ 242.664707] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 242.664755] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 242.664814] systemd-udevd D 00003fffa427aed4 0 173 159 0x00040002 [ 242.664886] Call Trace: [ 242.664907] [c00000001fe5f8a0] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 242.664968] [c00000001fe5f900] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 242.665027] [c00000001fe5f9e0] [c000000008b456cc] schedule+0x4c/0xc0 [ 242.665086] [c00000001fe5fa10] [c0000000080fcea4] async_synchronize_cookie_domain+0xb4/0x220 [ 242.665155] [c00000001fe5fad0] [c000000008b53c00] do_init_module+0x118/0x264 [ 242.665214] [c00000001fe5fb60] [c0000000081896e8] load_module+0x16a8/0x1870 [ 242.665273] [c00000001fe5fd30] [c000000008189bf0] SyS_finit_module+0xf0/0x170 [ 242.665333] [c00000001fe5fe30] [c000000008009584] system_call+0x38/0xec [ 242.665394] INFO: task kworker/u2:3:206 blocked for more than 120 seconds. [ 242.665443] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 242.665493] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 242.665550] kworker/u2:3 D 0000000000000000 0 206 2 0x00000800 [ 242.665621] Workqueue: scsi_tmf_1 scmd_eh_abort_handler [ 242.665670] Call Trace: [ 242.665691] [c00000001e3b76a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 242.665762] [c00000001e3b7870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 242.665822] [c00000001e3b78d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 242.665883] [c00000001e3b79b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 242.665942] [c00000001e3b79e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 242.666010] [c00000001e3b7ad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 242.666070] [c00000001e3b7b50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 242.666150] [c00000001e3b7bc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 242.666217] [c00000001e3b7c50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 242.666287] [c00000001e3b7ce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 242.666346] [c00000001e3b7d80] [c0000000080f8224] kthread+0x114/0x140 [ 242.666405] [c00000001e3b7e30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c done. Begin: Waiting for root file system ... Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. [ 363.495503] INFO: task kworker/u2:0:6 blocked for more than 120 seconds. [ 363.495579] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 363.495627] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 363.495684] kworker/u2:0 D 0000000000000000 0 6 2 0x00000800 [ 363.495782] Workqueue: scsi_tmf_0 scmd_eh_abort_handler [ 363.495839] Call Trace: [ 363.495874] [c00000002607f6a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 363.495951] [c00000002607f870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 363.496017] [c00000002607f8d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 363.496077] [c00000002607f9b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 363.496135] [c00000002607f9e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 363.496203] [c00000002607fad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 363.496283] [c00000002607fb50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 363.496362] [c00000002607fbc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 363.496436] [c00000002607fc50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 363.496504] [c00000002607fce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 363.496564] [c00000002607fd80] [c0000000080f8224] kthread+0x114/0x140 [ 363.496623] [c00000002607fe30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c [ 363.496735] INFO: task systemd-udevd:173 blocked for more than 120 seconds. [ 363.496787] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 363.496834] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 363.496890] systemd-udevd D 00003fffa427aed4 0 173 159 0x00040002 [ 363.496949] Call Trace: [ 363.496969] [c00000001fe5f8a0] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 363.497027] [c00000001fe5f900] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 363.497085] [c00000001fe5f9e0] [c000000008b456cc] schedule+0x4c/0xc0 [ 363.497143] [c00000001fe5fa10] [c0000000080fcea4] async_synchronize_cookie_domain+0xb4/0x220 [ 363.497209] [c00000001fe5fad0] [c000000008b53c00] do_init_module+0x118/0x264 [ 363.497267] [c00000001fe5fb60] [c0000000081896e8] load_module+0x16a8/0x1870 [ 363.497325] [c00000001fe5fd30] [c000000008189bf0] SyS_finit_module+0xf0/0x170 [ 363.497383] [c00000001fe5fe30] [c000000008009584] system_call+0x38/0xec [ 363.497443] INFO: task kworker/u2:3:206 blocked for more than 120 seconds. [ 363.497493] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 363.497540] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 363.497596] kworker/u2:3 D 0000000000000000 0 206 2 0x00000800 [ 363.497657] Workqueue: scsi_tmf_1 scmd_eh_abort_handler [ 363.497706] Call Trace: [ 363.497727] [c00000001e3b76a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 363.497810] [c00000001e3b7870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 363.497871] [c00000001e3b78d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 363.497929] [c00000001e3b79b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 363.497988] [c00000001e3b79e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 363.498054] [c00000001e3b7ad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 363.498112] [c00000001e3b7b50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 363.498188] [c00000001e3b7bc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 363.498254] [c00000001e3b7c50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 363.498321] [c00000001e3b7ce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 363.498379] [c00000001e3b7d80] [c0000000080f8224] kthread+0x114/0x140 [ 363.498438] [c00000001e3b7e30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. Begin: Running /scripts/local-block ... done. done. Gave up waiting for root device. Common problems: - Boot args (cat /proc/cmdline) - Check rootdelay= (did the system wait long enough?) - Check root= (did the system wait for the right device?) - Missing modules (cat /proc/modules; ls /dev) ALERT! UUID=a557092e-9d7f-496e-a371-7a043e192ec9 does not exist. Dropping to a shell! BusyBox v1.22.1 (Ubuntu 1:1.22.0-15ubuntu1) built-in shell (ash) Enter 'help' for a list of built-in commands. (initramfs) [ 484.327523] INFO: task kworker/u2:0:6 blocked for more than 120 seconds. [ 484.327625] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 484.327709] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 484.327808] kworker/u2:0 D 0000000000000000 0 6 2 0x00000800 [ 484.327961] Workqueue: scsi_tmf_0 scmd_eh_abort_handler [ 484.328060] Call Trace: [ 484.328117] [c00000002607f6a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 484.328247] [c00000002607f870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 484.328362] [c00000002607f8d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 484.328464] [c00000002607f9b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 484.328565] [c00000002607f9e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 484.328683] [c00000002607fad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 484.328809] [c00000002607fb50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 484.328943] [c00000002607fbc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 484.329072] [c00000002607fc50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 484.329189] [c00000002607fce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 484.329291] [c00000002607fd80] [c0000000080f8224] kthread+0x114/0x140 [ 484.329393] [c00000002607fe30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c [ 484.329555] INFO: task systemd-udevd:173 blocked for more than 120 seconds. [ 484.329640] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 484.329723] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 484.329822] systemd-udevd D 00003fffa427aed4 0 173 159 0x00040002 [ 484.329925] Call Trace: [ 484.329961] [c00000001fe5f8a0] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 484.330062] [c00000001fe5f900] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 484.330162] [c00000001fe5f9e0] [c000000008b456cc] schedule+0x4c/0xc0 [ 484.330265] [c00000001fe5fa10] [c0000000080fcea4] async_synchronize_cookie_domain+0xb4/0x220 [ 484.330382] [c00000001fe5fad0] [c000000008b53c00] do_init_module+0x118/0x264 [ 484.330483] [c00000001fe5fb60] [c0000000081896e8] load_module+0x16a8/0x1870 [ 484.330584] [c00000001fe5fd30] [c000000008189bf0] SyS_finit_module+0xf0/0x170 [ 484.330685] [c00000001fe5fe30] [c000000008009584] system_call+0x38/0xec [ 484.330803] INFO: task kworker/u2:3:206 blocked for more than 120 seconds. [ 484.330892] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 484.330974] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 484.331073] kworker/u2:3 D 0000000000000000 0 206 2 0x00000800 [ 484.331178] Workqueue: scsi_tmf_1 scmd_eh_abort_handler [ 484.331263] Call Trace: [ 484.331300] [c00000001e3b76a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 484.331418] [c00000001e3b7870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 484.331526] [c00000001e3b78d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 484.331629] [c00000001e3b79b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 484.331731] [c00000001e3b79e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 484.331850] [c00000001e3b7ad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 484.331954] [c00000001e3b7b50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 484.332089] [c00000001e3b7bc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 484.332210] [c00000001e3b7c50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 484.332330] [c00000001e3b7ce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 484.332435] [c00000001e3b7d80] [c0000000080f8224] kthread+0x114/0x140 [ 484.332540] [c00000001e3b7e30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c [ 605.159500] INFO: task kworker/u2:0:6 blocked for more than 120 seconds. [ 605.159599] Not tainted 4.8.0-32-generic #34~16.04.1-Ubuntu [ 605.159696] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 605.159799] kworker/u2:0 D 0000000000000000 0 6 2 0x00000800 [ 605.159906] Workqueue: scsi_tmf_0 scmd_eh_abort_handler [ 605.159990] Call Trace: [ 605.160027] [c00000002607f6a0] [c00000000964c930] logbuf_lock+0x0/0x4 (unreliable) [ 605.160147] [c00000002607f870] [c00000000801a6f0] __switch_to+0x2a0/0x420 [ 605.160249] [c00000002607f8d0] [c000000008b44fe4] __schedule+0x2f4/0x990 [ 605.160350] [c00000002607f9b0] [c000000008b456cc] schedule+0x4c/0xc0 [ 605.160452] [c00000002607f9e0] [c000000008b49ce4] schedule_timeout+0x284/0x480 [ 605.160570] [c00000002607fad0] [c000000008b4669c] wait_for_common+0xec/0x240 [ 605.160674] [c00000002607fb50] [d00000000063212c] virtscsi_tmf.constprop.4+0xcc/0x160 [virtio_scsi] [ 605.160806] [c00000002607fbc0] [c0000000087bf04c] scmd_eh_abort_handler+0x14c/0x390 [ 605.160923] [c00000002607fc50] [c0000000080ef5e8] process_one_work+0x1e8/0x5b0 [ 605.161039] [c00000002607fce0] [c0000000080efa58] worker_thread+0xa8/0x650 [ 605.161139] [c00000002607fd80] [c0000000080f8224] kthread+0x114/0x140 [ 605.161240] [c00000002607fe30] [c0000000080098f0] ret_from_kernel_thread+0x5c/0x6c root@10:~# exit Script done on Tue 20 Dec 2016 10:00:58 AM CST