InnoDB: Assertion failure in thread 139920848815872 in file buf0buf.cc line 4702 InnoDB: Failing assertion: rw_lock_is_locked(&block->lock, RW_LOCK_EX)

[Raghavendra D Prabhu]

Crashed during sysbench with auto-inc off.

Console: http://jenkins.percona.com/job/PXC-5.6-msysbench/185/BTYPE=debug,Host=centos7-64/console

Logs: http://jenkins.percona.com/job/PXC-5.6-msysbench/185/BTYPE=debug,Host=centos7-64/artifact/results-185.tar.gz

Backtrace
=========
#0 0x00007f41f193a771 in pthread_kill () from /lib64/libpthread.so.0
#1 0x0000000000ae5cfe in my_write_core (sig=6) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/mysys/stacktrace.c:422
#2 0x0000000000752c47 in handle_fatal_signal (sig=6) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/signal_handler.cc:236
#3 <signal handler called>
#4 0x00007f41efcd75c9 in raise () from /lib64/libc.so.6
#5 0x00007f41efcd8cd8 in abort () from /lib64/libc.so.6
#6 0x0000000000d5538a in buf_pool_validate_instance (buf_pool=0x2bcda68) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/buf/buf0buf.cc:4701
#7 0x0000000000d55a69 in buf_validate () at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/buf/buf0buf.cc:4864
#8 0x0000000000d51779 in buf_page_get_gen (space=11, zip_size=0, offset=12, rw_latch=1, guess=0x0, mode=10,
    file=0x109b388 "/mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/btr/btr0cur.cc", line=3872, mtr=0x7f41dc7cd030)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/buf/buf0buf.cc:3091
#9 0x0000000000d2c687 in btr_cur_search_to_nth_level (index=0x7f41901511b8, level=0, tuple=0x7f417c0cd408, mode=1, latch_mode=1, cursor=0x7f41dc7ccfb0, has_search_latch=0,
    file=0x109b388 "/mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/btr/btr0cur.cc", line=3872, mtr=0x7f41dc7cd030)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/btr/btr0cur.cc:652
#10 0x0000000000d33642 in btr_estimate_n_rows_in_range (index=0x7f41901511b8, tuple1=0x7f417c0cd3b8, mode1=2, tuple2=0x7f417c0cd408, mode2=1)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/btr/btr0cur.cc:3872
#11 0x0000000000b70651 in ha_innobase::records_in_range (this=0x7f417407f590, keynr=0, min_key=0x7f41dc7d14c0, max_key=0x7f41dc7d14e0)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/handler/ha_innodb.cc:12311
#12 0x0000000000668809 in handler::multi_range_read_info_const (this=0x7f417407f590, keyno=0, seq=0x7f41dc7d1650, seq_init_param=0x7f41dc7d1690, n_ranges_arg=0, bufsz=0x7f41dc7d15a0, flags=0x7f41dc7d15a4,
    cost=0x7f41dc7d19e0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/handler.cc:6060
#13 0x0000000000669ec4 in DsMrr_impl::dsmrr_info_const (this=0x7f417407fa28, keyno=0, seq=0x7f41dc7d1650, seq_init_param=0x7f41dc7d1690, n_ranges=0, bufsz=0x7f41dc7d1ae0, flags=0x7f41dc7d1ae4,
    cost=0x7f41dc7d19e0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/handler.cc:6702
#14 0x0000000000b7ae3b in ha_innobase::multi_range_read_info_const (this=0x7f417407f590, keyno=0, seq=0x7f41dc7d1650, seq_init_param=0x7f41dc7d1690, n_ranges=0, bufsz=0x7f41dc7d1ae0, flags=0x7f41dc7d1ae4,
    cost=0x7f41dc7d19e0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/storage/innobase/handler/ha_innodb.cc:19698
#15 0x00000000009a81f0 in check_quick_select (param=0x7f41dc7d1e40, idx=0, index_only=false, tree=0x7f417c0d5488, update_tbl_stats=true, mrr_flags=0x7f41dc7d1ae4, bufsize=0x7f41dc7d1ae0, cost=0x7f41dc7d19e0)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/opt_range.cc:9503
#16 0x000000000099fef6 in get_key_scans_params (param=0x7f41dc7d1e40, tree=0x7f417c0d4f28, index_read_must_be_used=false, update_tbl_stats=true, read_time=117.5)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/opt_range.cc:5600
#17 0x0000000000999d44 in SQL_SELECT::test_quick_select (this=0x7f417c0cdfb0, thd=0x2deb1f0, keys_to_use=..., prev_tables=0, limit=18446744073709551615, force_quick_range=false,
    interesting_order=st_order::ORDER_NOT_RELEVANT) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/opt_range.cc:2900
#18 0x00000000009f79b7 in get_quick_record_count (thd=0x2deb1f0, select=0x7f417c0cdfb0, table=0x7f4174085cb0, keys=0x7f417c002040, limit=18446744073709551615)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_optimizer.cc:4001
#19 0x00000000009f6cd3 in make_join_statistics (join=0x7f417c001980, tables_arg=0x7f417c000b28, conds=0x7f417c001478, keyuse_array=0x7f417c001ba8, first_optimization=true)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_optimizer.cc:3712
#20 0x00000000009ee28d in JOIN::optimize (this=0x7f417c001980) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_optimizer.cc:382
#21 0x000000000083c522 in mysql_execute_select (thd=0x2deb1f0, select_lex=0x2dee038, free_join=true) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_select.cc:1086
#22 0x000000000083c873 in mysql_select (thd=0x2deb1f0, tables=0x7f417c000b28, wild_num=0, fields=..., conds=0x7f417c001478, order=0x2dee200, group=0x2dee138, having=0x0, select_options=2148797184,
    result=0x7f417c001958, unit=0x2ded9f0, select_lex=0x2dee038) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_select.cc:1221
#23 0x000000000083a90a in handle_select (thd=0x2deb1f0, result=0x7f417c001958, setup_tables_done_option=0)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_select.cc:110
#24 0x00000000008122df in execute_sqlcom_select (thd=0x2deb1f0, all_tables=0x7f417c000b28) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:6059
#25 0x0000000000808f7b in mysql_execute_command (thd=0x2deb1f0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:3254
#26 0x000000000081536e in mysql_parse (thd=0x2deb1f0, rawbuf=0x7f417c0008d0 "SELECT c FROM sbtest6 WHERE id BETWEEN 4982 AND 4982+99 ORDER BY c", length=66, parser_state=0x7f41dc7d6580)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:7348
#27 0x00000000008147eb in wsrep_mysql_parse (thd=0x2deb1f0, rawbuf=0x7f417c0008d0 "SELECT c FROM sbtest6 WHERE id BETWEEN 4982 AND 4982+99 ORDER BY c", length=66, parser_state=0x7f41dc7d6580)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:7099
#28 0x00000000008052a2 in dispatch_command (command=COM_QUERY, thd=0x2deb1f0, packet=0x2e0a741 "SELECT c FROM sbtest6 WHERE id BETWEEN 4982 AND 4982+99 ORDER BY c", packet_length=66)
    at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:1625
#29 0x0000000000803b4a in do_command (thd=0x2deb1f0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_parse.cc:1129
#30 0x00000000007cb27c in do_handle_one_connection (thd_arg=0x2deb1f0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_connect.cc:1557
#31 0x00000000007cad71 in handle_one_connection (arg=0x2deb1f0) at /mnt/workspace/build-xtradb-cluster-binaries-56/BUILD_TYPE/debug/label_exp/centos7-64/sql/sql_connect.cc:1461
#32 0x00007f41f1935df3 in start_thread () from /lib64/libpthread.so.0
#33 0x00007f41efd9801d in clone () from /lib64/libc.so.6

[Raghavendra D Prabhu (raghavendra-prabhu)]

http://paste.wnohang.net/a72035 is the full log.

[Krunal Bauskar (krunal-bauskar)]

Haven't seen this since then. May be some bug got resolved either in PXC or PS as crash seems to be coming from PS codebase.
Will close it now and revisit it if observed again.

[Shahriyar Rzayev (rzayev-sehriyar)]

Percona now uses JIRA for bug reports so this bug report is migrated to: https://jira.percona.com/browse/PXC-1019