2013-06-20 15:26:25 7f85c81fc700 InnoDB: Assertion failure in thread 140212564903680 in file log0log.cc line 308
InnoDB: Failing assertion: ++count < 50
Thread 1 (LWP 29188):
+bt
#0 0x0000003fdde328a5 in raise () from /lib64/libc.so.6
#1 0x0000003fdde34085 in abort () from /lib64/libc.so.6
#2 0x0000000000a35520 in log_open (len=<optimized out>) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/log/log0log.cc:308
#3 0x0000000000a4940e in mtr_log_reserve_and_write (mtr=0x7f85ccf67840) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/mtr/mtr0mtr.cc:278
#4 mtr_commit (mtr=0x7f85ccf67840) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/mtr/mtr0mtr.cc:323
#5 0x0000000000b3f273 in btr_store_big_rec_extern_fields (index=0x7f8578048898, rec_block=<optimized out>, rec=0x7f85d43bb49d "\200", offsets=0x7f8578076c90, big_rec_vec=0x7f8578061fc8, btr_mtr=0x7f85ccf67e10, op=BTR_STORE_INSERT) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/btr/btr0cur.cc:4973
#6 0x0000000000a939ee in row_ins_index_entry_big_rec_func (entry=0x7f85d43bb49d, big_rec=0x7f8578061fc8, offsets=0x7f8578076c90, heap=0x7f85ccf68c98, index=0x7f8578048898, file=<optimized out>, thd=0x36c4f80, line=2529) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:2840
#7 0x0000000000a97f07 in row_ins_clust_index_entry_low (flags=0, mode=2, index=0x7f8578048898, n_uniq=<optimized out>, entry=0x7f8578045cd8, n_ext=0, thr=0x7f85780419b0) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:2529
#8 0x0000000000a9a075 in row_ins_clust_index_entry (index=0x7f8578048898, entry=0x7f8578045cd8, thr=0x7f85780419b0, n_ext=0) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:2887
#9 0x0000000000a9b7f7 in row_ins_index_entry (thr=0x7f85780419b0, entry=0x7f8578045cd8, index=0x7f8578048898) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:2981
#10 row_ins_index_entry_step (thr=0x7f85780419b0, node=0x7f85780414c8) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:3058
#11 row_ins (thr=0x7f85780419b0, node=0x7f85780414c8) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:3198
#12 row_ins_step (thr=0x7f85780419b0) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0ins.cc:3323
#13 0x0000000000aae1ec in row_insert_for_mysql (mysql_rec=<optimized out>, prebuilt=<optimized out>) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/row/row0mysql.cc:1312
#14 0x00000000009ea39e in ha_innobase::write_row (this=0x7f857803dd50, record=0x7f8578047b90 "") at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/innobase/handler/ha_innodb.cc:6929
#15 0x00000000005b7b04 in handler::ha_write_row (this=0x7f857803dd50, buf=0x7f8578047b90 "") at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/handler.cc:7414
#16 0x00000000006f8540 in write_record (thd=0x36c4f80, table=0x7f857803c4c0, info=0x7f8578005a70, update=0x7f8578005ae8) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_insert.cc:1905
#17 0x00000000006f9035 in select_insert::send_data (this=0x7f8578005a30, values=...) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_insert.cc:3681
#18 0x00000000006eacf8 in end_send (join=0x7f8578005b68, join_tab=0x7f85780504e8, end_of_records=false) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_executor.cc:2775
#19 0x00000000006ec2d3 in evaluate_join_record (join=0x7f8578005b68, join_tab=0x7f85780501e8) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_executor.cc:1598
#20 0x00000000006f09b6 in sub_select (join=0x7f8578005b68, join_tab=0x7f85780501e8, end_of_records=<optimized out>) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_executor.cc:1273
#21 0x00000000006f22e6 in do_select (join=0x7f8578005b68) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_executor.cc:930
#22 JOIN::exec (this=0x7f8578005b68) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_executor.cc:191
#23 0x000000000073c800 in mysql_execute_select (free_join=true, select_lex=0x36c7a48, thd=0x36c4f80) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_select.cc:1101
#24 mysql_select (thd=0x36c4f80, tables=0x7f85780054b8, wild_num=1, fields=..., conds=0x0, order=<optimized out>, group=0x36c7b48, having=0x0, select_options=3489925888, result=0x7f8578005a30, unit=0x36c7400, select_lex=0x36c7a48) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_select.cc:1222
#25 0x000000000073d2c3 in handle_select (thd=0x36c4f80, result=0x7f8578005a30, setup_tables_done_option=1073741824) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_select.cc:111
#26 0x0000000000719dc4 in mysql_execute_command (thd=0x36c4f80) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_parse.cc:3714
#27 0x000000000071b8b8 in mysql_parse (thd=0x36c4f80, rawbuf=0x36c7338 "p\234\065\001", length=<optimized out>, parser_state=<optimized out>) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_parse.cc:6451
#28 0x000000000071c923 in dispatch_command (command=COM_QUERY, thd=0x36c4f80, packet=0x3628c31 " INSERT INTO z SELECT * FROM `table500_innodb_default_int` ", packet_length=59) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_parse.cc:1376
#29 0x000000000071d86b in do_command (thd=0x36c4f80) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_parse.cc:1051
#30 0x00000000006e261f in do_handle_one_connection (thd_arg=<optimized out>) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_connect.cc:1615
#31 0x00000000006e2707 in handle_one_connection (arg=0x36c4f80) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/sql/sql_connect.cc:1526
#32 0x00000000009beb49 in pfs_spawn_thread (arg=0x3673a80) at /bzr/5.6_dbg/Percona-Server-5.6.11-rc60.3/storage/perfschema/pfs.cc:1853
#33 0x0000003fde207851 in start_thread () from /lib64/libpthread.so.0
#34 0x0000003fddee890d in clone () from /lib64/libc.so.6
Aborts in other threads also, see attaches
[roel@qaserver 402661]$ cat cmd215 cmdrun_ 215 cmdrun_ 215 column_ names --reporter= Shutdown, Backtrace, QueryTimeout, ErrorLog, ErrorLogAlarm --mysqld= --log-output= none --mysqld= --sql_mode= ONLY_FULL_ GROUP_BY --grammar= conf/percona_ qa/5.6/ 5.6.yy --gendata= conf/percona_ qa/5.6/ 5.6.zz --basedir= /ssd2/Percona- Server- 5.6.11- rc60.3- 383-debug. Linux.x86_ 64 --threads=25 --no-mask --notnull --validator= Transformer --mysqld= --innodb_ track_changed_ pages=1 --mysqld= --innodb_ max_bitmap_ file_size= 4097 --mysqld= --innodb_ changed_ pages=ON --mysqld= --innodb_ max_changed_ pages=2 --mysqld= --slow_ query_log --mysqld=--userstat --mysqld= --innodb_ log_archive= 1 --mysqld= --innodb_ log_arch_ dir=/ssd2/ 402661/ _epoch/ 137173100579322 1699 --mysqld= --innodb_ log_arch_ expire_ sec=120 --mysqld= --innodb_ log_file_ size=1048576 --mysqld= --innodb_ log_files_ in_group= 10 --mysqld= --innodb_ log_buffer_ size=10485761 --mysqld= --innodb_ flush_log_ at_trx_ commit= 2 --mysqld= --query_ cache_type= 1 --mysqld= --query_ cache_size= 1048576 --mysqld= --innodb_ flush_method= O_DIRECT --mysqld= --innodb_ file_per_ table=1 --mtr-build- thread= 794 --mask=24562 --vardir1= /ssd2/402661/ cmdrun_ 215 > /ssd2/402661/ cmdrun215. log 2>&1"
ps -ef | grep 'cmdrun_215' | grep -v grep | awk '{print $2}' | xargs sudo kill -9
rm -Rf /ssd2/402661/
mkdir /ssd2/402661/
cd /ssd2/randgen
bash -c "set -o pipefail; perl runall.pl --queries=100000000 --seed=734 --duration=300 --querytimeout=60 --short_