rpl.percona_bug1331586 'stmt' crashes debug build with "sql/rpl_slave.cc:4455: int exec_relay_log_event(THD*, Relay_log_info*): Assertion `thd->transaction.all.cannot_safely_rollback()' failed." intermittently

[Laurynas Biveinis]

On 5.6 trunk, http://jenkins.percona.com/job/percona-server-5.6-trunk/739/BUILD_TYPE=debug,Host=centos5-64/consoleText:

rpl.percona_bug1331586 'stmt' w2 [ fail ]
        Test ended at 2017-02-16 08:02:53

CURRENT_TEST: rpl.percona_bug1331586
mysqltest: In included file "./include/stop_slave.inc":
included from ./include/stop_slave.inc at line 82:
At line 82: query 'STOP SLAVE' failed: 2013: Lost connection to MySQL server during query
...
mysqld-debug: /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:4455: int exec_relay_log_event(THD*, Relay_log_info*): Assertion `thd->transaction.all.cannot_safely_rollback()' failed.
...
Thread 27 (Thread 0x7f7352bb6940 (LWP 12244)):
#0 0x00007f7364435019 in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000ad8b9e in safe_cond_wait (cond=0x32402d8, mp=0x3240268,
    file=0xfcc740 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h", line=1151)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/mysys/thr_mutex.c:240
#2 0x0000000000aa5bf8 in inline_mysql_cond_wait (that=0x32402d8,
    mutex=0x3240268,
    src_file=0xfcca48 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc", src_line=2020)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h:1151
#3 0x0000000000aab14a in pop_jobs_item (worker=0x323cfa0,
    job_item=0x7f7352bb5fa0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc:2020
#4 0x0000000000aab2e6 in slave_worker_exec_job (worker=0x323cfa0,
    rli=0x7f73480c24a0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc:2060
#5 0x0000000000a8d66b in handle_slave_worker (arg=0x323cfa0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:5303
#6 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x316c350)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#7 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#8 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 26 (Thread 0x7f7364946940 (LWP 12241)):
#0 0x00007f7362a120b6 in poll () from /lib64/libc.so.6
#1 0x0000000000e9c1ed in vio_io_wait (vio=0x7f73481df700,
    event=VIO_IO_EVENT_READ, timeout=28800000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:1137
#2 0x0000000000e9a6ee in vio_socket_io_wait (vio=0x7f73481df700,
    event=VIO_IO_EVENT_READ)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:104
#3 0x0000000000e9a7ce in vio_read (vio=0x7f73481df700,
    buf=0x7f73481df900 "\001", size=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:159
#4 0x000000000072d399 in net_read_raw_loop (net=0x7f73481db868, count=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:667
#5 0x000000000072d567 in net_read_packet_header (net=0x7f73481db868)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:749
#6 0x000000000072d666 in net_read_packet (net=0x7f73481db868,
    complen=0x7f7364945f30)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:814
#7 0x000000000072d88a in my_net_read (net=0x7f73481db868)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:892
#8 0x00000000007ed1a0 in do_command (thd=0x7f73481db5f0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:995
#9 0x00000000007b3eef in do_handle_one_connection (thd_arg=0x7f73481db5f0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1550
#10 0x00000000007b3bf6 in handle_one_connection (arg=0x7f73481db5f0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1454
#11 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x7f7348101f20)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#12 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#13 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 25 (Thread 0x7f7364905940 (LWP 12242)):
#0 0x00007f7362a120b6 in poll () from /lib64/libc.so.6
#1 0x0000000000e9c1ed in vio_io_wait (vio=0x3134c40,
    event=VIO_IO_EVENT_READ, timeout=120000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:1137
#2 0x0000000000e9a6ee in vio_socket_io_wait (vio=0x3134c40,
    event=VIO_IO_EVENT_READ)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:104
#3 0x0000000000e9a7ce in vio_read (vio=0x3134c40, buf=0x3336260 "1",
    size=16384)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:159
#4 0x0000000000e9a985 in vio_read_buff (vio=0x3134c40, buf=0x333a270 "",
    size=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:193
#5 0x000000000072d399 in net_read_raw_loop (net=0x3044900, count=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:667
#6 0x000000000072d599 in net_read_packet_header (net=0x3044900)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:755
#7 0x000000000072d666 in net_read_packet (net=0x3044900,
    complen=0x7f7364904de0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:814
#8 0x000000000072d88a in my_net_read (net=0x3044900)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:892
#9 0x00000000008db1ff in cli_safe_read (mysql=0x3044900)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql-common/client.c:620
#10 0x0000000000a89125 in read_event (mysql=0x3044900, mi=0x7f73480873e0,
    suppress_warnings=0x7f736490500b)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:3737
#11 0x0000000000a8c4e0 in handle_slave_io (arg=0x7f73480873e0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:4948
#12 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x31569a0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#13 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#14 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x7f7364987940 (LWP 12240)):
#0 0x00007f7362a120b6 in poll () from /lib64/libc.so.6
#1 0x0000000000e9c1ed in vio_io_wait (vio=0x7f73481d7290,
    event=VIO_IO_EVENT_READ, timeout=28800000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:1137
#2 0x0000000000e9a6ee in vio_socket_io_wait (vio=0x7f73481d7290,
    event=VIO_IO_EVENT_READ)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:104
#3 0x0000000000e9a7ce in vio_read (vio=0x7f73481d7290,
    buf=0x7f73481d7490 "\001", size=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:159
#4 0x000000000072d399 in net_read_raw_loop (net=0x7f73481d33f8, count=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:667
#5 0x000000000072d567 in net_read_packet_header (net=0x7f73481d33f8)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:749
#6 0x000000000072d666 in net_read_packet (net=0x7f73481d33f8,
    complen=0x7f7364986f30)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:814
#7 0x000000000072d88a in my_net_read (net=0x7f73481d33f8)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:892
#8 0x00000000007ed1a0 in do_command (thd=0x7f73481d3180)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:995
#9 0x00000000007b3eef in do_handle_one_connection (thd_arg=0x7f73481d3180)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1550
#10 0x00000000007b3bf6 in handle_one_connection (arg=0x7f73481d3180)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1454
#11 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x7f73480f0d70)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#12 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#13 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x7f7347fff940 (LWP 12232)):
#0 0x00007f7362a14172 in select () from /lib64/libc.so.6
#1 0x0000000000b9b351 in os_thread_sleep (tm=1000000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0thread.cc:304
#2 0x0000000000cf2ee6 in page_cleaner_sleep_if_needed (
    next_loop_time=1487250173623)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/buf/buf0flu.cc:2584
#3 0x0000000000cf31fb in buf_flush_page_cleaner_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/buf/buf0flu.cc:2714
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x7f7352b75940 (LWP 12245)):
#0 0x00007f7364435019 in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000ad8b9e in safe_cond_wait (cond=0x3297628, mp=0x32975b8,
    file=0xfcc740 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h", line=1151)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/mysys/thr_mutex.c:240
#2 0x0000000000aa5bf8 in inline_mysql_cond_wait (that=0x3297628,
    mutex=0x32975b8,
    src_file=0xfcca48 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc", src_line=2020)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h:1151
#3 0x0000000000aab14a in pop_jobs_item (worker=0x32942f0,
    job_item=0x7f7352b74fa0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc:2020
#4 0x0000000000aab2e6 in slave_worker_exec_job (worker=0x32942f0,
    rli=0x7f73480c24a0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_rli_pdb.cc:2060
#5 0x0000000000a8d66b in handle_slave_worker (arg=0x32942f0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:5303
#6 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x31686c0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#7 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#8 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x7f7353fb7940 (LWP 12199)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30bb990, segment=1,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=5,
    message1=0x7f7353fb7090, message2=0x7f7353fb7088, type=0x7f7353fb7080,
    space_id=0x7f7353fb7078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=5)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840648)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x7f735138d940 (LWP 12228)):
#0 0x00007f7362a14172 in select () from /lib64/libc.so.6
#1 0x0000000000b9b351 in os_thread_sleep (tm=1000000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0thread.cc:304
#2 0x0000000000c56f13 in srv_master_sleep ()
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:2945
#3 0x0000000000c57056 in srv_master_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:2995
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x7f73567b9940 (LWP 12197)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30852a0, segment=1,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=3,
    message1=0x7f73567b9090, message2=0x7f73567b9088, type=0x7f73567b9080,
    space_id=0x7f73567b9078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=3)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840638)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 18 (Thread 0x7f7364a09940 (LWP 12238)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000ad8d6b in safe_cond_timedwait (cond=0x7f73480c2af8,
    mp=0x7f73480c29a8, abstime=0x7f7364a06c70,
    file=0xfbfda0 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h", line=1188)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/mysys/thr_mutex.c:278
#2 0x0000000000a7e6a3 in inline_mysql_cond_timedwait (that=0x7f73480c2af8,
    mutex=0x7f73480c29a8, abstime=0x7f7364a06c70,
    src_file=0xfc0a18 "/mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc", src_line=1431)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/include/mysql/psi/mysql_thread.h:1188
#3 0x0000000000a8165f in terminate_slave_thread (thd=0x32226a0,
    term_lock=0x7f73480c29a8, term_cond=0x7f73480c2af8,
    slave_running=0x7f73480c2bac, need_lock_term=false)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:1431
#4 0x0000000000a80f57 in terminate_slave_threads (mi=0x7f73480873e0,
    thread_mask=3, need_lock_term=false)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:1260
#5 0x0000000000a96a26 in stop_slave (thd=0x7f73480e1680, mi=0x7f73480873e0,
    net_report=true)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:8783
#6 0x00000000007f38c2 in mysql_execute_command (thd=0x7f73480e1680)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:3663
#7 0x00000000007fcc21 in mysql_parse (thd=0x7f73480e1680,
    rawbuf=0x31811b0 "STOP SLAVE", length=10, parser_state=0x7f7364a08140)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:7056
#8 0x00000000007ee633 in dispatch_command (command=COM_QUERY,
    thd=0x7f73480e1680, packet=0x7f73481c6bb1 "STOP SLAVE", packet_length=10)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:1485
#9 0x00000000007ed411 in do_command (thd=0x7f73480e1680)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:1062
#10 0x00000000007b3eef in do_handle_one_connection (thd_arg=0x7f73480e1680)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1550
#11 0x00000000007b3bf6 in handle_one_connection (arg=0x7f73480e1680)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1454
#12 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x7f734812e300)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#13 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#14 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7f734eb8a940 (LWP 12227)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a69d in os_cond_wait_timed (cond=0x2f2e168,
    fast_mutex=0x2f2e130, abstime=0x7f734eb8a050)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:159
#2 0x0000000000b9acae in os_event_wait_time_low (event=0x2f2e130,
    time_in_usec=1000000, reset_sig_count=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:591
#3 0x0000000000c55763 in srv_error_monitor_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:2186
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7f7364a4b720 (LWP 12192)):
#0 0x00007f7362a120b6 in poll () from /lib64/libc.so.6
#1 0x0000000000632c4c in handle_connections_sockets ()
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/mysqld.cc:6704
#2 0x0000000000631c2a in mysqld_main (argc=90, argv=0x2f2d448)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/mysqld.cc:6112
#3 0x00000000006257e0 in main (argc=12, argv=0x7ffc5882e1c8)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/main.cc:25

Thread 15 (Thread 0x7f73553b8940 (LWP 12198)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30bb990, segment=0,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=4,
    message1=0x7f73553b8090, message2=0x7f73553b8088, type=0x7f73553b8080,
    space_id=0x7f73553b8078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840640)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x7f7357bba940 (LWP 12196)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30852a0, segment=0,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=2,
    message1=0x7f7357bba090, message2=0x7f7357bba088, type=0x7f7357bba080,
    space_id=0x7f7357bba078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=2)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840630)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x7f73443fc940 (LWP 12235)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a69d in os_cond_wait_timed (cond=0x312e118,
    fast_mutex=0x312e0e0, abstime=0x7f73443fc0b0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:159
#2 0x0000000000b9acae in os_event_wait_time_low (event=0x312e0e0,
    time_in_usec=10000000, reset_sig_count=3)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:591
#3 0x0000000000d3d3ef in dict_stats_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/dict/dict0stats_bg.cc:349
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7f73457fd940 (LWP 12234)):
#0 0x00007f7364435019 in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a785 in os_cond_wait (cond=0x303ade8, fast_mutex=0x303adb0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:196
#2 0x0000000000b9aab8 in os_event_wait_low (event=0x303adb0,
    reset_sig_count=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:477
#3 0x0000000000ceb043 in buf_dump_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/buf/buf0dump.cc:613
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7f7346bfe940 (LWP 12233)):
#0 0x00007f7362a14172 in select () from /lib64/libc.so.6
#1 0x0000000000b9b351 in os_thread_sleep (tm=999000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0thread.cc:304
#2 0x0000000000cf2ee6 in page_cleaner_sleep_if_needed (
    next_loop_time=1487250173618)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/buf/buf0flu.cc:2584
#3 0x0000000000cf379b in buf_flush_lru_manager_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/buf/buf0flu.cc:2862
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7f73649c8940 (LWP 12239)):
#0 0x00007f7362a120b6 in poll () from /lib64/libc.so.6
#1 0x0000000000e9c1ed in vio_io_wait (vio=0x7f73481cee20,
    event=VIO_IO_EVENT_READ, timeout=28800000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:1137
#2 0x0000000000e9a6ee in vio_socket_io_wait (vio=0x7f73481cee20,
    event=VIO_IO_EVENT_READ)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:104
#3 0x0000000000e9a7ce in vio_read (vio=0x7f73481cee20,
    buf=0x7f73481cf020 "\001", size=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/vio/viosocket.c:159
#4 0x000000000072d399 in net_read_raw_loop (net=0x7f73481caf88, count=4)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:667
#5 0x000000000072d567 in net_read_packet_header (net=0x7f73481caf88)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:749
#6 0x000000000072d666 in net_read_packet (net=0x7f73481caf88,
    complen=0x7f73649c7f30)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:814
#7 0x000000000072d88a in my_net_read (net=0x7f73481caf88)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/net_serv.cc:892
#8 0x00000000007ed1a0 in do_command (thd=0x7f73481cad10)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_parse.cc:995
#9 0x00000000007b3eef in do_handle_one_connection (thd_arg=0x7f73481cad10)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1550
#10 0x00000000007b3bf6 in handle_one_connection (arg=0x7f73481cad10)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/sql_connect.cc:1454
#11 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x7f734812e300)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#12 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#13 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7f734d789940 (LWP 12229)):
#0 0x00007f7364435019 in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a785 in os_cond_wait (cond=0x2f2e0f8, fast_mutex=0x2f2e0c0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:196
#2 0x0000000000b9aab8 in os_event_wait_low (event=0x2f2e0c0,
    reset_sig_count=3)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:477
#3 0x0000000000c57a0b in srv_purge_coordinator_suspend (slot=0x3061ac8,
    rseg_history_len=6)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:3305
#4 0x0000000000c57f1d in srv_purge_coordinator_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:3427
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7f7342ffb940 (LWP 12236)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a69d in os_cond_wait_timed (cond=0x7f734803b538,
    fast_mutex=0x7f734803b500, abstime=0x7f7342ffb000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:159
#2 0x0000000000b9acae in os_event_wait_time_low (event=0x7f734803b500,
    time_in_usec=5000000, reset_sig_count=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:591
#3 0x0000000000c99cf6 in ib_wqueue_timedwait (wq=0x7f734803b368,
    wait_in_usecs=5000000)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/ut/ut0wqueue.cc:154
#4 0x0000000000d7be61 in fts_optimize_thread (arg=0x7f734803b368)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fts/fts0opt.cc:3059
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7f7364a4a940 (LWP 12237)):
#0 0x00007f73644389c8 in do_sigwait () from /lib64/libpthread.so.0
#1 0x00007f7364438a6d in sigwait () from /lib64/libpthread.so.0
#2 0x000000000062cd7e in signal_hand (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/mysqld.cc:3488
#3 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x7f734805f870)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7f735a3bc940 (LWP 12194)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30adf40, segment=0,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=0,
    message1=0x7f735a3bc090, message2=0x7f735a3bc088, type=0x7f735a3bc080,
    space_id=0x7f735a3bc078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840620)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7f734ff8b940 (LWP 12226)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a69d in os_cond_wait_timed (cond=0x3120798,
    fast_mutex=0x3120760, abstime=0x7f734ff8b090)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:159
#2 0x0000000000b9acae in os_event_wait_time_low (event=0x3120760,
    time_in_usec=1000000, reset_sig_count=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:591
#3 0x0000000000b70cef in lock_wait_timeout_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/lock/lock0wait.cc:503
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7f7358fbb940 (LWP 12195)):
#0 0x00007f73642295b4 in ?? () from /usr/lib64/libaio.so.1
#1 0x0000000000b98198 in os_aio_linux_collect (array=0x30a04f0, segment=0,
    seg_size=256)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5201
#2 0x0000000000b9852c in os_aio_linux_handle (global_seg=1,
    message1=0x7f7358fbb090, message2=0x7f7358fbb088, type=0x7f7358fbb080,
    space_id=0x7f7358fbb078)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0file.cc:5347
#3 0x0000000000d4e9c6 in fil_aio_wait (segment=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/fil/fil0fil.cc:5907
#4 0x0000000000c5a9d0 in io_handler_thread (arg=0x1840628)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0start.cc:515
#5 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#6 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7f735278e940 (LWP 12200)):
#0 0x00007f7364435280 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
   from /lib64/libpthread.so.0
#1 0x0000000000b9a69d in os_cond_wait_timed (cond=0x303ad78,
    fast_mutex=0x303ad40, abstime=0x7f735278e060)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:159
#2 0x0000000000b9acae in os_event_wait_time_low (event=0x303ad40,
    time_in_usec=5000000, reset_sig_count=1)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/os/os0sync.cc:591
#3 0x0000000000c5515e in srv_monitor_thread (arg=0x0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/innobase/srv/srv0srv.cc:1953
#4 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#5 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7f735c4c0940 (LWP 12193)):
#0 0x00007f73629774a4 in do_sigwaitinfo () from /lib64/libc.so.6
#1 0x00007f736297755d in sigwaitinfo () from /lib64/libc.so.6
#2 0x0000000000adc6f0 in timer_notify_thread (arg=0x7ffc5882dab0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/mysys/posix_timers.c:78
#3 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#4 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f73648c4940 (LWP 12243)):
#0 0x00007f7364435e32 in pthread_kill () from /lib64/libpthread.so.0
#1 0x0000000000ad478a in my_write_core (sig=6)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/mysys/stacktrace.c:424
#2 0x000000000073b133 in handle_fatal_signal (sig=6)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/signal_handler.cc:236
#3 <signal handler called>
#4 0x00007f7362975fc5 in raise () from /lib64/libc.so.6
#5 0x00007f7362977a70 in abort () from /lib64/libc.so.6
#6 0x00007f736296f466 in __assert_fail () from /lib64/libc.so.6
#7 0x0000000000a8adc8 in exec_relay_log_event (thd=0x32226a0,
    rli=0x7f73480c24a0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:4444
#8 0x0000000000a909a8 in handle_slave_sql (arg=0x7f73480873e0)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/sql/rpl_slave.cc:6413
#9 0x0000000000e3bcb1 in pfs_spawn_thread (arg=0x3134e40)
    at /mnt/workspace/percona-server-5.6-trunk/BUILD_TYPE/debug/Host/centos5-64/storage/perfschema/pfs.cc:1860
#10 0x00007f736443083d in start_thread () from /lib64/libpthread.so.0
#11 0x00007f7362a1afdd in clone () from /lib64/libc.so.6

[Shahriyar Rzayev (rzayev-sehriyar)]

Percona now uses JIRA for bug reports so this bug report is migrated to: https://jira.percona.com/browse/PS-3648