From the previous backtrace, this is what the gist is: (annotated with where it spins) Thread 8 (Thread 0x7f2317f0e700 (LWP 16676)): #0 0x00007f233adc61fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x00000000008f6285 in os_cond_wait (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:207 #2 os_event_wait_low (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:609 #3 0x000000000082dab2 in sync_array_wait_event (arr=0x3037d30, index=2) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0arr.c:458 #4 0x000000000082ecfc in mutex_spin_wait (mutex=0x7f2327c848c0, file_name=, line=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0sync.c:653 #5 0x000000000086f19e in mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa83d80 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c", line=3667) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:222 #6 pfs_mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa83d80 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c", line=3667) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:251 #7 0x00000000008773ae in buf_page_init_for_read (err=, mode=, space=108, zip_size=, unzip=, tablespace_version=, offset=1771977) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c:3667 >>>> buf_pool_mutex_enter(buf_pool); buf_pool->n_pend_reads++; buf_pool_mutex_exit(buf_pool); >>>> #8 0x0000000000885b14 in buf_read_page_low (err=0x7f2317f0d028, sync=, mode=, space=108, zip_size=0, unzip=0, tablespace_version=124, offset=1771977, trx=0x0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0rea.c:165 #9 0x00000000008865a5 in buf_read_ibuf_merge_pages (sync=, space_ids=, space_versions=, page_nos=0x7f2317f0d6a0, n_stored=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0rea.c:816 #10 0x00000000008c9440 in ibuf_contract_ext (n_pages=0x7f2317f0d728, sync=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/ibuf/ibuf0ibuf.c:2729 #11 0x00000000008c9527 in ibuf_contract_for_n_pages (sync=0, n_pages=10) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/ibuf/ibuf0ibuf.c:2775 #12 0x0000000000825460 in srv_master_thread (arg=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/srv/srv0srv.c:3341 #13 0x00007f233adc18ca in start_thread () from /lib/libpthread.so.0 #14 0x00007f233a06292d in clone () from /lib/libc.so.6 #15 0x0000000000000000 in ?? () Thread 7 (Thread 0x7f231770d700 (LWP 16677)): #0 0x00007f233adc61fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x00000000008f6285 in os_cond_wait (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:207 #2 os_event_wait_low (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:609 #3 0x000000000082dab2 in sync_array_wait_event (arr=0x3037d30, index=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0arr.c:458 #4 0x000000000082ecfc in mutex_spin_wait (mutex=0x7f2327c848c0, file_name=, line=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0sync.c:653 #5 0x000000000087e4fe in mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa857f0 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0lru.c", line=1102) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:222 #6 pfs_mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa857f0 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0lru.c", line=1102) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:251 #7 0x00000000008853d8 in buf_LRU_try_free_flushed_blocks (buf_pool=0x7f2327c848c0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0lru.c:1102 >>>>> buf_pool_mutex_enter(buf_pool); while (buf_pool->LRU_flush_ended > 0) { buf_pool_mutex_exit(buf_pool); buf_LRU_search_and_free_block(buf_pool, 1); >>>>>> #8 0x00000000008854d6 in buf_LRU_try_free_flushed_blocks (buf_pool=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0lru.c:1099 #9 0x0000000000863dd8 in ibuf_should_try (index=0x7f2327e9a0c0, level=, tuple=, mode=, latch_mode=, cursor=, has_search_latch=0, file=0xa7a110 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0row.c", line=797, mtr=0x7f231770c620) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/ibuf0ibuf.ic:138 #10 btr_cur_search_to_nth_level (index=0x7f2327e9a0c0, level=, tuple=, mode=, latch_mode=, cursor=, has_search_latch=0, file=0xa7a110 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0row.c", line=797, mtr=0x7f231770c620) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/btr/btr0cur.c:609 #11 0x0000000000811487 in btr_pcur_open_func (index=0x35b5a14, entry=0x7f23280a3920, mode=8194, pcur=0x7f231770cad0, mtr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/btr0pcur.ic:438 #12 row_search_index_entry (index=0x35b5a14, entry=0x7f23280a3920, mode=8194, pcur=0x7f231770cad0, mtr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0row.c:797 #13 0x0000000000915b8e in row_purge_remove_sec_if_poss_leaf (node=0x7f2327cc5a40, index=0x7f2327e9a0c0, entry=0x7f23280a3920) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0purge.c:375 #14 0x0000000000916f1a in row_purge_remove_sec_if_poss (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0purge.c:433 #15 row_purge_del_mark (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0purge.c:484 #16 row_purge (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0purge.c:765 #17 row_purge_step (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0purge.c:805 #18 0x000000000090b52f in que_thr_step (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/que/que0que.c:1259 #19 que_run_threads_low (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/que/que0que.c:1319 #20 que_run_threads (thr=0x7f2327cc5988) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/que/que0que.c:1356 #21 0x0000000000832057 in trx_purge (limit=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/trx/trx0purge.c:1195 #22 0x000000000082918c in srv_purge_thread (arg=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/srv/srv0srv.c:3967 #23 0x00007f233adc18ca in start_thread () from /lib/libpthread.so.0 #24 0x00007f233a06292d in clone () from /lib/libc.so.6 #25 0x0000000000000000 in ?? () Thread 4 (Thread 0x7f233b01e700 (LWP 16691)): #0 0x00007f233adc61fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x00000000008f6285 in os_cond_wait (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:207 #2 os_event_wait_low (event=0x35b59d0, reset_sig_count=1) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/os/os0sync.c:609 #3 0x000000000082dab2 in sync_array_wait_event (arr=0x3037d30, index=0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0arr.c:458 #4 0x000000000082ecfc in mutex_spin_wait (mutex=0x7f2327c848c0, file_name=, line=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/sync/sync0sync.c:653 #5 0x000000000086f19e in mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa83d80 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c", line=3667) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:222 #6 pfs_mutex_enter_func (mutex=0x7f2327c848c0, file_name=0xa83d80 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c", line=3667) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/include/sync0sync.ic:251 #7 0x00000000008773ae in buf_page_init_for_read (err=, mode=, space=108, zip_size=, unzip=, tablespace_version=, offset=1603180) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c:3667 >>>>>>> buf_pool_mutex_enter(buf_pool); buf_pool->n_pend_reads++; buf_pool_mutex_exit(buf_pool); >>>>>>> #8 0x0000000000885b14 in buf_read_page_low (err=0x7f233b01a808, sync=, mode=, space=108, zip_size=0, unzip=0, tablespace_version=124, offset=1603180, trx=0x0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0rea.c:165 #9 0x0000000000886670 in buf_read_page (space=108, zip_size=0, offset=1603180, trx=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0rea.c:452 #10 0x0000000000875fcb in buf_page_get_gen (space=, zip_size=, offset=, rw_latch=, guess=0x0, mode=, file=0xa98d78 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c", line=2025, mtr=0x7f233b01b3a0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/buf/buf0buf.c:2590 #11 0x0000000000863366 in btr_cur_search_to_nth_level (index=0x7f2327eb40c0, level=, tuple=, mode=, latch_mode=, cursor=, has_search_latch=0, file=0xa98d78 "/home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c", line=2025, mtr=0x7f233b01b3a0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/btr/btr0cur.c:623 #12 0x00000000009130cf in row_ins_index_entry_low (mode=2, index=0x7f2327eb40c0, entry=0x7f2327ffe8c0, n_ext=, thr=0x7f23280000c0) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c:2023 #13 0x000000000091567f in row_ins_index_entry (thr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c:2278 #14 row_ins_index_entry_step (thr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c:2363 #15 row_ins (thr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c:2495 #16 row_ins_step (thr=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0ins.c:2612 #17 0x0000000000809896 in row_insert_for_mysql (mysql_rec=, prebuilt=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/row/row0mysql.c:1255 #18 0x00000000007f3f81 in ha_innobase::write_row (this=0x3de2c70, record=0x3de48a0 "") at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/storage/innobase/handler/ha_innodb.cc:5911 #19 0x000000000069b4ff in handler::ha_write_row (this=0x3de2c70, buf=0x3de48a0 "") at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/handler.cc:5334 #20 0x00000000005803d0 in write_record (thd=0x3a67830, table=0x3de2390, info=0x7f233b01c340) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/sql_insert.cc:1779 #21 0x0000000000586b03 in mysql_insert (thd=0x3a67830, table_list=0x3a77048, fields=, values_list=, update_fields=, update_values=, duplic=DUP_ERROR, ignore=false) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/sql_insert.cc:956 #22 0x00000000005971c8 in mysql_execute_command (thd=0x3a67830) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/sql_parse.cc:3065 #23 0x000000000059a853 in mysql_parse (thd=0x3a67830, rawbuf=, length=, parser_state=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/sql_parse.cc:5811 #24 0x0000000000745144 in Query_log_event::do_apply_event (this=0x7f23100fdb70, rli=0x3a53aa8, query_arg=, q_len_arg=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/log_event.cc:3471 #25 0x0000000000533815 in Log_event::apply_event (ev=0x7f23100fdb70, thd=, rli=0x3a53aa8) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/log_event.h:1143 #26 apply_event_and_update_pos (ev=0x7f23100fdb70, thd=, rli=0x3a53aa8) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/slave.cc:2400 #27 0x0000000000538359 in exec_relay_log_event (thd=, rli=0x3a53aa8) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/slave.cc:2560 #28 0x0000000000539799 in handle_slave_sql (arg=) at /home/jenkins/workspace/percona-server-5.5-debs/label_exp/debian6-64/target/Percona-Server-5.5.27-rel28.0/sql/slave.cc:3385 #29 0x00007f233adc18ca in start_thread () from /lib/libpthread.so.0 #30 0x00007f233a06292d in clone () from /lib/libc.so.6 #31 0x0000000000000000 in ?? () ============================================= So, all three threads are waitin on buf_pool mutex. One of the threads (out of all) must be holding that mutex and sleeping or trying it recursively.