Got some more stack traces: [22201.670647] INFO: task mongod:1917 blocked for more than 120 seconds. [22201.670650] Tainted: P O 4.4.0-34-generic #53-Ubuntu [22201.670651] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [22201.670653] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [22201.670655] ffff8805aecefba8 00000000579b1d9a ffff8807faf06040 ffff8806d6238000 [22201.670657] ffff8805aecf0000 ffff88081edd6d00 7fffffffffffffff ffffffff8182a610 [22201.670659] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [22201.670661] Call Trace: [22201.670666] [] ? bit_wait+0x60/0x60 [22201.670668] [] schedule+0x35/0x80 [22201.670670] [] schedule_timeout+0x1b5/0x270 [22201.670673] [] ? find_get_pages_tag+0x109/0x190 [22201.670675] [] ? bit_wait+0x60/0x60 [22201.670676] [] io_schedule_timeout+0xa4/0x110 [22201.670678] [] bit_wait_io+0x1b/0x70 [22201.670680] [] __wait_on_bit+0x5d/0x90 [22201.670682] [] wait_on_page_bit+0xcb/0xf0 [22201.670684] [] ? autoremove_wake_function+0x40/0x40 [22201.670686] [] __filemap_fdatawait_range+0xf3/0x160 [22201.670688] [] ? __filemap_fdatawrite_range+0xd1/0x100 [22201.670690] [] filemap_fdatawait_range+0x14/0x30 [22201.670691] [] filemap_write_and_wait_range+0x3f/0x70 [22201.670732] [] zpl_fsync+0x38/0x90 [zfs] [22201.670734] [] vfs_fsync_range+0x4b/0xb0 [22201.670736] [] SyS_msync+0x17e/0x1f0 [22201.670738] [] entry_SYSCALL_64_fastpath+0x16/0x71 [85684.344389] INFO: task mongod:1917 blocked for more than 120 seconds. [85684.344393] Tainted: P O 4.4.0-34-generic #53-Ubuntu [85684.344394] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [85684.344395] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [85684.344398] ffff8805aecefba8 0000000000000003 ffffffff81e11500 ffff8806d6238000 [85684.344400] ffff8805aecf0000 ffff88081ec16d00 7fffffffffffffff ffffffff8182a610 [85684.344402] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [85684.344403] Call Trace: [85684.344409] [] ? bit_wait+0x60/0x60 [85684.344411] [] schedule+0x35/0x80 [85684.344413] [] schedule_timeout+0x1b5/0x270 [85684.344416] [] ? find_get_pages_tag+0x109/0x190 [85684.344418] [] ? bit_wait+0x60/0x60 [85684.344419] [] io_schedule_timeout+0xa4/0x110 [85684.344421] [] bit_wait_io+0x1b/0x70 [85684.344423] [] __wait_on_bit+0x5d/0x90 [85684.344425] [] wait_on_page_bit+0xcb/0xf0 [85684.344427] [] ? autoremove_wake_function+0x40/0x40 [85684.344429] [] __filemap_fdatawait_range+0xf3/0x160 [85684.344431] [] ? __filemap_fdatawrite_range+0xd1/0x100 [85684.344432] [] filemap_fdatawait_range+0x14/0x30 [85684.344434] [] filemap_write_and_wait_range+0x3f/0x70 [85684.344477] [] zpl_fsync+0x38/0x90 [zfs] [85684.344479] [] vfs_fsync_range+0x4b/0xb0 [85684.344481] [] SyS_msync+0x17e/0x1f0 [85684.344483] [] entry_SYSCALL_64_fastpath+0x16/0x71 [110165.396049] INFO: task mongod:1917 blocked for more than 120 seconds. [110165.396053] Tainted: P O 4.4.0-34-generic #53-Ubuntu [110165.396054] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [110165.396055] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [110165.396058] ffff8805aecefba8 00000000579c74f7 ffff8807faf02940 ffff8806d6238000 [110165.396060] ffff8805aecf0000 ffff88081ecd6d00 7fffffffffffffff ffffffff8182a610 [110165.396062] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [110165.396064] Call Trace: [110165.396070] [] ? bit_wait+0x60/0x60 [110165.396072] [] schedule+0x35/0x80 [110165.396074] [] schedule_timeout+0x1b5/0x270 [110165.396076] [] ? find_get_pages_tag+0x109/0x190 [110165.396078] [] ? bit_wait+0x60/0x60 [110165.396080] [] io_schedule_timeout+0xa4/0x110 [110165.396082] [] bit_wait_io+0x1b/0x70 [110165.396083] [] __wait_on_bit+0x5d/0x90 [110165.396085] [] wait_on_page_bit+0xcb/0xf0 [110165.396088] [] ? autoremove_wake_function+0x40/0x40 [110165.396090] [] __filemap_fdatawait_range+0xf3/0x160 [110165.396091] [] ? __filemap_fdatawrite_range+0xd1/0x100 [110165.396093] [] filemap_fdatawait_range+0x14/0x30 [110165.396094] [] filemap_write_and_wait_range+0x3f/0x70 [110165.396137] [] zpl_fsync+0x38/0x90 [zfs] [110165.396139] [] vfs_fsync_range+0x4b/0xb0 [110165.396141] [] SyS_msync+0x17e/0x1f0 [110165.396143] [] entry_SYSCALL_64_fastpath+0x16/0x71 [132726.355047] INFO: task mongod:1917 blocked for more than 120 seconds. [132726.355062] Tainted: P O 4.4.0-34-generic #53-Ubuntu [132726.355063] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [132726.355065] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [132726.355067] ffff8805aecefba8 0000000000000003 ffff8806d623e040 ffff8806d6238000 [132726.355069] ffff8805aecf0000 ffff88081ed56d00 7fffffffffffffff ffffffff8182a610 [132726.355071] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [132726.355073] Call Trace: [132726.355079] [] ? bit_wait+0x60/0x60 [132726.355081] [] schedule+0x35/0x80 [132726.355082] [] schedule_timeout+0x1b5/0x270 [132726.355085] [] ? find_get_pages_tag+0x109/0x190 [132726.355087] [] ? bit_wait+0x60/0x60 [132726.355089] [] io_schedule_timeout+0xa4/0x110 [132726.355091] [] bit_wait_io+0x1b/0x70 [132726.355092] [] __wait_on_bit+0x5d/0x90 [132726.355094] [] wait_on_page_bit+0xcb/0xf0 [132726.355097] [] ? autoremove_wake_function+0x40/0x40 [132726.355099] [] __filemap_fdatawait_range+0xf3/0x160 [132726.355101] [] filemap_fdatawait_range+0x14/0x30 [132726.355102] [] filemap_write_and_wait_range+0x3f/0x70 [132726.355143] [] zpl_fsync+0x38/0x90 [zfs] [132726.355146] [] vfs_fsync_range+0x4b/0xb0 [132726.355148] [] SyS_msync+0x17e/0x1f0 [132726.355150] [] entry_SYSCALL_64_fastpath+0x16/0x71 [152887.214613] INFO: task mongod:1917 blocked for more than 120 seconds. [152887.214617] Tainted: P O 4.4.0-34-generic #53-Ubuntu [152887.214618] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [152887.214619] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [152887.214622] ffff8805aecefba8 00000000579d1c18 ffff8807faf01b80 ffff8806d6238000 [152887.214624] ffff8805aecf0000 ffff88081ec96d00 7fffffffffffffff ffffffff8182a610 [152887.214626] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [152887.214628] Call Trace: [152887.214634] [] ? bit_wait+0x60/0x60 [152887.214636] [] schedule+0x35/0x80 [152887.214637] [] schedule_timeout+0x1b5/0x270 [152887.214640] [] ? find_get_pages_tag+0x109/0x190 [152887.214642] [] ? bit_wait+0x60/0x60 [152887.214644] [] io_schedule_timeout+0xa4/0x110 [152887.214646] [] bit_wait_io+0x1b/0x70 [152887.214647] [] __wait_on_bit+0x5d/0x90 [152887.214649] [] wait_on_page_bit+0xcb/0xf0 [152887.214652] [] ? autoremove_wake_function+0x40/0x40 [152887.214654] [] __filemap_fdatawait_range+0xf3/0x160 [152887.214656] [] filemap_fdatawait_range+0x14/0x30 [152887.214657] [] filemap_write_and_wait_range+0x3f/0x70 [152887.214698] [] zpl_fsync+0x38/0x90 [zfs] [152887.214700] [] vfs_fsync_range+0x4b/0xb0 [152887.214703] [] SyS_msync+0x17e/0x1f0 [152887.214704] [] entry_SYSCALL_64_fastpath+0x16/0x71 [170287.952597] INFO: task txg_sync:4602 blocked for more than 120 seconds. [170287.952600] Tainted: P O 4.4.0-34-generic #53-Ubuntu [170287.952602] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [170287.952603] txg_sync D ffff8800d95c3aa8 0 4602 2 0x00000000 [170287.952606] ffff8800d95c3aa8 ffff88081edd6d00 ffff8807faf06040 ffff8807f627ee00 [170287.952608] ffff8800d95c4000 ffff88081edd6d00 7fffffffffffffff ffff88017c6cc0a8 [170287.952610] 0000000000000001 ffff8800d95c3ac0 ffffffff81829e15 0000000000000000 [170287.952611] Call Trace: [170287.952617] [] schedule+0x35/0x80 [170287.952619] [] schedule_timeout+0x1b5/0x270 [170287.952621] [] ? set_next_entity+0x9c/0xb0 [170287.952624] [] ? __switch_to+0x1dc/0x5c0 [170287.952626] [] io_schedule_timeout+0xa4/0x110 [170287.952633] [] cv_wait_common+0xbc/0x140 [spl] [170287.952635] [] ? wake_atomic_t_function+0x60/0x60 [170287.952640] [] __cv_wait_io+0x18/0x20 [spl] [170287.952683] [] zio_wait+0x10e/0x1f0 [zfs] [170287.952704] [] dsl_pool_sync+0xb8/0x430 [zfs] [170287.952726] [] spa_sync+0x366/0xb30 [zfs] [170287.952729] [] ? default_wake_function+0x12/0x20 [170287.952753] [] txg_sync_thread+0x3ba/0x630 [zfs] [170287.952778] [] ? txg_delay+0x180/0x180 [zfs] [170287.952782] [] thread_generic_wrapper+0x71/0x80 [spl] [170287.952785] [] ? __thread_exit+0x20/0x20 [spl] [170287.952787] [] kthread+0xd8/0xf0 [170287.952789] [] ? kthread_create_on_node+0x1e0/0x1e0 [170287.952791] [] ret_from_fork+0x3f/0x70 [170287.952792] [] ? kthread_create_on_node+0x1e0/0x1e0 [170287.952873] INFO: task mongod:1917 blocked for more than 120 seconds. [170287.952874] Tainted: P O 4.4.0-34-generic #53-Ubuntu [170287.952875] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [170287.952876] mongod D ffff8805aecefaa8 0 1917 5819 0x00000100 [170287.952877] ffff8805aecefaa8 ffff8805aecefa80 ffff8807faf05280 ffff8806d6238000 [170287.952879] ffff8805aecf0000 ffff88081ed96d00 7fffffffffffffff ffffffff8182a610 [170287.952881] ffff8805aecefc08 ffff8805aecefac0 ffffffff81829e15 0000000000000000 [170287.952882] Call Trace: [170287.952884] [] ? bit_wait+0x60/0x60 [170287.952886] [] schedule+0x35/0x80 [170287.952887] [] schedule_timeout+0x1b5/0x270 [170287.952910] [] ? rrm_exit+0x48/0x80 [zfs] [170287.952937] [] ? zfs_putpage+0x524/0x560 [zfs] [170287.952939] [] ? bit_wait+0x60/0x60 [170287.952941] [] io_schedule_timeout+0xa4/0x110 [170287.952942] [] bit_wait_io+0x1b/0x70 [170287.952944] [] __wait_on_bit+0x5d/0x90 [170287.952950] [] wait_on_page_bit+0xcb/0xf0 [170287.952951] [] ? autoremove_wake_function+0x40/0x40 [170287.952954] [] write_cache_pages+0x392/0x510 [170287.952981] [] ? zpl_write_common_iovec+0xe0/0xe0 [zfs] [170287.952982] [] ? mutex_lock+0x12/0x30 [170287.953004] [] ? rrw_exit+0x60/0x150 [zfs] [170287.953031] [] zpl_writepages+0x103/0x170 [zfs] [170287.953033] [] do_writepages+0x1e/0x30 [170287.953034] [] __filemap_fdatawrite_range+0xc6/0x100 [170287.953036] [] filemap_write_and_wait_range+0x2a/0x70 [170287.953062] [] zpl_fsync+0x38/0x90 [zfs] [170287.953065] [] vfs_fsync_range+0x4b/0xb0 [170287.953067] [] SyS_msync+0x17e/0x1f0 [170287.953069] [] entry_SYSCALL_64_fastpath+0x16/0x71 [170287.953177] INFO: task kworker/u16:3:31765 blocked for more than 120 seconds. [170287.953178] Tainted: P O 4.4.0-34-generic #53-Ubuntu [170287.953179] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [170287.953179] kworker/u16:3 D ffff880749d47838 0 31765 2 0x00000000 [170287.953183] Workqueue: writeback wb_workfn (flush-zfs-37) [170287.953184] ffff880749d47838 ffffffff811ec63a ffff8807faf01b80 ffff88048c348000 [170287.953186] ffff880749d48000 ffff88081ec96d00 7fffffffffffffff ffff880749d479a8 [170287.953187] ffffffff8182a610 ffff880749d47850 ffffffff81829e15 0000000000000000 [170287.953189] Call Trace: [170287.953191] [] ? kfree+0x13a/0x150 [170287.953193] [] ? bit_wait+0x60/0x60 [170287.953195] [] schedule+0x35/0x80 [170287.953196] [] schedule_timeout+0x1b5/0x270 [170287.953221] [] ? rrm_exit+0x48/0x80 [zfs] [170287.953224] [] ? ktime_get+0x3c/0xb0 [170287.953226] [] ? bit_wait+0x60/0x60 [170287.953228] [] io_schedule_timeout+0xa4/0x110 [170287.953229] [] bit_wait_io+0x1b/0x70 [170287.953231] [] __wait_on_bit_lock+0x4e/0xb0 [170287.953233] [] __lock_page+0xbb/0xe0 [170287.953235] [] ? autoremove_wake_function+0x40/0x40 [170287.953237] [] write_cache_pages+0x2dd/0x510 [170287.953239] [] ? lock_timer_base.isra.22+0x54/0x70 [170287.953265] [] ? zpl_write_common_iovec+0xe0/0xe0 [zfs] [170287.953292] [] zpl_writepages+0x94/0x170 [zfs] [170287.953294] [] do_writepages+0x1e/0x30 [170287.953296] [] __writeback_single_inode+0x45/0x330 [170287.953298] [] writeback_sb_inodes+0x262/0x600 [170287.953299] [] __writeback_inodes_wb+0x8c/0xc0 [170287.953301] [] wb_writeback+0x253/0x310 [170287.953303] [] ? get_nr_dirty_inodes+0x51/0x80 [170287.953305] [] wb_workfn+0x2ea/0x400 [170287.953307] [] process_one_work+0x165/0x480 [170287.953309] [] worker_thread+0x4b/0x4c0 [170287.953310] [] ? process_one_work+0x480/0x480 [170287.953312] [] kthread+0xd8/0xf0 [170287.953313] [] ? kthread_create_on_node+0x1e0/0x1e0 [170287.953315] [] ret_from_fork+0x3f/0x70 [170287.953316] [] ? kthread_create_on_node+0x1e0/0x1e0 [184208.557065] INFO: task mongod:1917 blocked for more than 120 seconds. [184208.557068] Tainted: P O 4.4.0-34-generic #53-Ubuntu [184208.557069] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [184208.557071] mongod D ffff8805aecefba8 0 1917 5819 0x00000100 [184208.557074] ffff8805aecefba8 00000000579d9674 ffff8807faf05280 ffff8806d6238000 [184208.557076] ffff8805aecf0000 ffff88081ed96d00 7fffffffffffffff ffffffff8182a610 [184208.557077] ffff8805aecefd08 ffff8805aecefbc0 ffffffff81829e15 0000000000000000 [184208.557079] Call Trace: [184208.557085] [] ? bit_wait+0x60/0x60 [184208.557087] [] schedule+0x35/0x80 [184208.557089] [] schedule_timeout+0x1b5/0x270 [184208.557092] [] ? find_get_pages_tag+0x109/0x190 [184208.557094] [] ? bit_wait+0x60/0x60 [184208.557096] [] io_schedule_timeout+0xa4/0x110 [184208.557097] [] bit_wait_io+0x1b/0x70 [184208.557099] [] __wait_on_bit+0x5d/0x90 [184208.557101] [] wait_on_page_bit+0xcb/0xf0 [184208.557104] [] ? autoremove_wake_function+0x40/0x40 [184208.557106] [] __filemap_fdatawait_range+0xf3/0x160 [184208.557107] [] ? __filemap_fdatawrite_range+0xd1/0x100 [184208.557109] [] filemap_fdatawait_range+0x14/0x30 [184208.557110] [] filemap_write_and_wait_range+0x3f/0x70 [184208.557151] [] zpl_fsync+0x38/0x90 [zfs] [184208.557154] [] vfs_fsync_range+0x4b/0xb0 [184208.557156] [] SyS_msync+0x17e/0x1f0 [184208.557158] [] entry_SYSCALL_64_fastpath+0x16/0x71 [201369.291288] INFO: task txg_sync:4602 blocked for more than 120 seconds. [201369.291292] Tainted: P O 4.4.0-34-generic #53-Ubuntu [201369.291293] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [201369.291294] txg_sync D ffff8800d95c3aa8 0 4602 2 0x00000000 [201369.291298] ffff8800d95c3aa8 ffff88081ed96d00 ffff8807faf05280 ffff8807f627ee00 [201369.291300] ffff8800d95c4000 ffff88081ed96d00 7fffffffffffffff ffff880485c17988 [201369.291301] 0000000000000001 ffff8800d95c3ac0 ffffffff81829e15 0000000000000000 [201369.291303] Call Trace: [201369.291309] [] schedule+0x35/0x80 [201369.291311] [] schedule_timeout+0x1b5/0x270 [201369.291313] [] ? set_next_entity+0x9c/0xb0 [201369.291316] [] ? __switch_to+0x1dc/0x5c0 [201369.291318] [] io_schedule_timeout+0xa4/0x110 [201369.291325] [] cv_wait_common+0xbc/0x140 [spl] [201369.291327] [] ? wake_atomic_t_function+0x60/0x60 [201369.291331] [] __cv_wait_io+0x18/0x20 [spl] [201369.291369] [] zio_wait+0x10e/0x1f0 [zfs] [201369.291389] [] dsl_pool_sync+0xb8/0x430 [zfs] [201369.291411] [] spa_sync+0x366/0xb30 [zfs] [201369.291414] [] ? default_wake_function+0x12/0x20 [201369.291438] [] txg_sync_thread+0x3ba/0x630 [zfs] [201369.291461] [] ? txg_delay+0x180/0x180 [zfs] [201369.291465] [] thread_generic_wrapper+0x71/0x80 [spl] [201369.291469] [] ? __thread_exit+0x20/0x20 [spl] [201369.291470] [] kthread+0xd8/0xf0 [201369.291472] [] ? kthread_create_on_node+0x1e0/0x1e0 [201369.291474] [] ret_from_fork+0x3f/0x70 [201369.291475] [] ? kthread_create_on_node+0x1e0/0x1e0