Hello. After upgrade to 10.2.2-0ubuntu1, they are still crashing on the same command: -1> 2016-06-30 14:43:11.973392 7f4f48fc2700 0 mon.pp6@1(leader).osd e10230 moving crush item name 'pp1' to location {root=fast2500} 0> 2016-06-30 14:43:11.975532 7f4f48fc2700 -1 crush/CrushWrapper.h: In function 'int CrushWrapper::detach_bucket(CephContext*, int)' thread 7f4f48fc2700 time 2016-06-30 14:43:11.973438 crush/CrushWrapper.h: 940: FAILED assert(successful_detach) ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x80) [0x559f04e3d7c0] 2: (()+0x5680f3) [0x559f04d4b0f3] 3: (CrushWrapper::move_bucket(CephContext*, int, std::map, std::allocator >, std::__cxx11::basic_string, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > const, std::__cxx11::basic_string, std::allocator > > > > const&)+0xda) [0x559f04d48d0a] 4: (OSDMonitor::prepare_command_impl(std::shared_ptr, std::map, std::allocator >, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > >, std::less, std::allocator > >, std::allocator, std::allocator > const, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > > > > >&)+0x2cfe) [0x559f04b0a4be] 5: (OSDMonitor::prepare_command(std::shared_ptr)+0x2ff) [0x559f04b1c51f] 6: (OSDMonitor::prepare_update(std::shared_ptr)+0x24b) [0x559f04b1ca6b] 7: (PaxosService::dispatch(std::shared_ptr)+0xb4f) [0x559f04acf31f] 8: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908] 9: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2] 10: (Context::complete(int)+0x9) [0x559f04a97b99] 11: (void finish_contexts(CephContext*, std::__cxx11::list >&, int)+0x1fb) [0x559f04a9e4cb] 12: (Paxos::finish_round()+0x287) [0x559f04ac4d87] 13: (Paxos::handle_last(std::shared_ptr)+0xe19) [0x559f04ac5f69] 14: (Paxos::dispatch(std::shared_ptr)+0x250) [0x559f04ac6790] 15: (Monitor::dispatch_op(std::shared_ptr)+0xa38) [0x559f04a920b8] 16: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4] 17: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963] 18: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb] 19: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d] 20: (()+0x76fa) [0x7f4f50d066fa] 21: (clone()+0x6d) [0x7f4f4efbab5d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 1/ 5 newstore 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 1/ 5 kinetic 1/ 5 fuse -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-mon.pp6.log --- end dump of recent events --- 2016-06-30 14:43:11.978616 7f4f48fc2700 -1 *** Caught signal (Aborted) ** in thread 7f4f48fc2700 thread_name:ms_dispatch ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (()+0x52ac7e) [0x559f04d0dc7e] 2: (()+0x113d0) [0x7f4f50d103d0] 3: (gsignal()+0x38) [0x7f4f4eee9418] 4: (abort()+0x16a) [0x7f4f4eeeb01a] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x26b) [0x559f04e3d9ab] 6: (()+0x5680f3) [0x559f04d4b0f3] 7: (CrushWrapper::move_bucket(CephContext*, int, std::map, std::allocator >, std::__cxx11::basic_string, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > const, std::__cxx11::basic_string, std::allocator > > > > const&)+0xda) [0x559f04d48d0a] 8: (OSDMonitor::prepare_command_impl(std::shared_ptr, std::map, std::allocator >, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > >, std::less, std::allocator > >, std::allocator, std::allocator > const, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > > > > >&)+0x2cfe) [0x559f04b0a4be] 9: (OSDMonitor::prepare_command(std::shared_ptr)+0x2ff) [0x559f04b1c51f] 10: (OSDMonitor::prepare_update(std::shared_ptr)+0x24b) [0x559f04b1ca6b] 11: (PaxosService::dispatch(std::shared_ptr)+0xb4f) [0x559f04acf31f] 12: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908] 13: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2] 14: (Context::complete(int)+0x9) [0x559f04a97b99] 15: (void finish_contexts(CephContext*, std::__cxx11::list >&, int)+0x1fb) [0x559f04a9e4cb] 16: (Paxos::finish_round()+0x287) [0x559f04ac4d87] 17: (Paxos::handle_last(std::shared_ptr)+0xe19) [0x559f04ac5f69] 18: (Paxos::dispatch(std::shared_ptr)+0x250) [0x559f04ac6790] 19: (Monitor::dispatch_op(std::shared_ptr)+0xa38) [0x559f04a920b8] 20: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4] 21: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963] 22: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb] 23: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d] 24: (()+0x76fa) [0x7f4f50d066fa] 25: (clone()+0x6d) [0x7f4f4efbab5d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- 0> 2016-06-30 14:43:11.978616 7f4f48fc2700 -1 *** Caught signal (Aborted) ** in thread 7f4f48fc2700 thread_name:ms_dispatch ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (()+0x52ac7e) [0x559f04d0dc7e] 2: (()+0x113d0) [0x7f4f50d103d0] 3: (gsignal()+0x38) [0x7f4f4eee9418] 4: (abort()+0x16a) [0x7f4f4eeeb01a] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x26b) [0x559f04e3d9ab] 6: (()+0x5680f3) [0x559f04d4b0f3] 7: (CrushWrapper::move_bucket(CephContext*, int, std::map, std::allocator >, std::__cxx11::basic_string, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > const, std::__cxx11::basic_string, std::allocator > > > > const&)+0xda) [0x559f04d48d0a] 8: (OSDMonitor::prepare_command_impl(std::shared_ptr, std::map, std::allocator >, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > >, std::less, std::allocator > >, std::allocator, std::allocator > const, boost::variant, std::allocator >, bool, long, double, std::vector, std::allocator >, std::allocator, std::allocator > > > > > > >&)+0x2cfe) [0x559f04b0a4be] 9: (OSDMonitor::prepare_command(std::shared_ptr)+0x2ff) [0x559f04b1c51f] 10: (OSDMonitor::prepare_update(std::shared_ptr)+0x24b) [0x559f04b1ca6b] 11: (PaxosService::dispatch(std::shared_ptr)+0xb4f) [0x559f04acf31f] 12: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908] 13: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2] 14: (Context::complete(int)+0x9) [0x559f04a97b99] 15: (void finish_contexts(CephContext*, std::__cxx11::list >&, int)+0x1fb) [0x559f04a9e4cb] 16: (Paxos::finish_round()+0x287) [0x559f04ac4d87] 17: (Paxos::handle_last(std::shared_ptr)+0xe19) [0x559f04ac5f69] 18: (Paxos::dispatch(std::shared_ptr)+0x250) [0x559f04ac6790] 19: (Monitor::dispatch_op(std::shared_ptr)+0xa38) [0x559f04a920b8] 20: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4] 21: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963] 22: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb] 23: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d] 24: (()+0x76fa) [0x7f4f50d066fa] 25: (clone()+0x6d) [0x7f4f4efbab5d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 1/ 5 newstore 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 1/ 5 kinetic 1/ 5 fuse -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-mon.pp6.log --- end dump of recent events ---