mysql> \s -------------- mysql Ver 14.14 Distrib 5.6.15, for Linux (x86_64) using EditLine wrapper Connection id: 6 Current database: test Current user: root@localhost SSL: Not in use Current pager: stdout Using outfile: '' Using delimiter: ; Server version: 5.6.15-63.0-log Percona XtraDB Cluster (GPL), Release 25.5, wsrep_25.5.r4061 Protocol version: 10 Connection: Localhost via UNIX socket Server characterset: latin1 Db characterset: latin1 Client characterset: utf8 Conn. characterset: utf8 UNIX socket: /var/run/mysqld/mysqld.sock Uptime: 12 min 23 sec Threads: 3 Question mysql> SET binlog_rows_query_log_events=1; Query OK, 0 rows affected (0.00 sec) mysql> SELECT @@session.binlog_rows_query_log_events; +----------------------------------------+ | @@session.binlog_rows_query_log_events | +----------------------------------------+ | 1 | +----------------------------------------+ 1 row in set (0.00 sec) mysql> SHOW CREATE TABLE foo\G *************************** 1. row *************************** Table: foo Create Table: CREATE TABLE `foo` ( `id` int(11) NOT NULL AUTO_INCREMENT, `username` varchar(64) NOT NULL, `a` varchar(32) NOT NULL, `b` varchar(16) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `username` (`username`) ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1 1 row in set (0.00 sec) mysql> INSERT INTO foo VALUES (NULL, 'bar', 'e', 'f'); Query OK, 1 row affected (0.00 sec) mysql> SHOW GLOBAL STATUS LIKE 'wsrep%'; +------------------------------+--------------------------------------+ | Variable_name | Value | +------------------------------+--------------------------------------+ | wsrep_local_state_uuid | 2ef7daec-c640-11e3-8375-7f735d3781b1 | | wsrep_protocol_version | 5 | | wsrep_last_committed | 18 | | wsrep_replicated | 4 | | wsrep_replicated_bytes | 1215 | | wsrep_repl_keys | 14 | | wsrep_repl_keys_bytes | 204 | | wsrep_repl_data_bytes | 755 | | wsrep_repl_other_bytes | 0 | | wsrep_received | 8 | | wsrep_received_bytes | 767 | | wsrep_local_commits | 3 | | wsrep_local_cert_failures | 0 | | wsrep_local_replays | 0 | | wsrep_local_send_queue | 0 | | wsrep_local_send_queue_avg | 0.000000 | | wsrep_local_recv_queue | 0 | | wsrep_local_recv_queue_avg | 0.000000 | | wsrep_local_cached_downto | 15 | | wsrep_flow_control_paused_ns | 0 | | wsrep_flow_control_paused | 0.000000 | | wsrep_flow_control_sent | 0 | | wsrep_flow_control_recv | 0 | | wsrep_cert_deps_distance | 1.250000 | | wsrep_apply_oooe | 0.000000 | | wsrep_apply_oool | 0.000000 | | wsrep_apply_window | 1.000000 | | wsrep_commit_oooe | 0.000000 | | wsrep_commit_oool | 0.000000 | | wsrep_commit_window | 1.000000 | | wsrep_local_state | 0 | | wsrep_local_state_comment | Initialized | | wsrep_cert_index_size | 7 | | wsrep_causal_reads | 0 | | wsrep_incoming_addresses | 192.168.0.100:3306 | | wsrep_cluster_conf_id | 18446744073709551615 | | wsrep_cluster_size | 1 | | wsrep_cluster_state_uuid | 2ef7daec-c640-11e3-8375-7f735d3781b1 | | wsrep_cluster_status | non-Primary | | wsrep_connected | ON | | wsrep_local_bf_aborts | 0 | | wsrep_local_index | 0 | | wsrep_provider_name | Galera | | wsrep_provider_vendor | Codership Oy | | wsrep_provider_version | 3.4(r176) | | wsrep_ready | OFF | +------------------------------+--------------------------------------+ 46 rows in set (0.00 sec) mysql> \! mysqlbinlog -vv /binlogs/mysql-bin.000009 | less # at 636 #140418 14:02:04 server id 10405113 end_log_pos 713 CRC32 0x503dd672 Query thread_id=6 exec_time=0 error_code=0 SET TIMESTAMP=1397847724/*!*/; SET @@session.auto_increment_increment=2, @@session.auto_increment_offset=1/*!*/; BEGIN /*!*/; # at 713 #140418 14:02:04 server id 10405113 end_log_pos 783 CRC32 0xf3e7976d Rows_query # INSERT INTO foo VALUES (NULL, 'bar', 'e', 'f') # at 783 #140418 14:02:04 server id 10405113 end_log_pos 838 CRC32 0xfece3853 Table_map: `test`.`foo` mapped to number 71 # at 838 #140418 14:02:04 server id 10405113 end_log_pos 886 CRC32 0x4bbe75d0 Write_rows: table id 71 flags: STMT_END_F BINLOG ' rHZRUx35xJ4ARgAAAA8DAACAAC5JTlNFUlQgSU5UTyBmb28gVkFMVUVTIChOVUxMLCAnYmFyJywg J2UnLCAnZicpbZfn8w== rHZRUxP5xJ4ANwAAAEYDAAAAAEcAAAAAAAEABHRlc3QAA2ZvbwAEAw8PDwZAACAAEAAAUzjO/g== rHZRUx75xJ4AMAAAAHYDAAAAAEcAAAAAAAEAAgAE//ABAAAAA2JhcgFlAWbQdb5L '/*!*/; ### INSERT INTO `test`.`foo` ### SET ### @1=1 /* INT meta=0 nullable=0 is_null=0 */ ### @2='bar' /* VARSTRING(64) meta=64 nullable=0 is_null=0 */ ### @3='e' /* VARSTRING(32) meta=32 nullable=0 is_null=0 */ ### @4='f' /* VARSTRING(16) meta=16 nullable=0 is_null=0 */ # at 886 #140418 14:02:04 server id 10405113 end_log_pos 917 CRC32 0x75a1ad55 Xid = 18 COMMIT/*!*/; error log from event: 2014-04-18 13:55:29 3767 [Note] WSREP: Member 1 (galera2) synced with group. 2014-04-18 14:02:04 3767 [Note] WSREP: (eb5e6359-c72a-11e3-bac7-db36affbd3b8, 'tcp://0.0.0.0:4567') turning message relay request ing on, nonlive peers: tcp://192.168.0.101:4567 2014-04-18 14:02:05 3767 [Note] WSREP: (eb5e6359-c72a-11e3-bac7-db36affbd3b8, 'tcp://0.0.0.0:4567') reconnecting to f9449da4-c72a -11e3-8ef5-f3b521c818d8 (tcp://192.168.0.101:4567), attempt 0 2014-04-18 14:02:09 3767 [Note] WSREP: evs::proto(eb5e6359-c72a-11e3-bac7-db36affbd3b8, OPERATIONAL, view_id(REG,eb5e6359-c72a-11 e3-bac7-db36affbd3b8,2)) suspecting node: f9449da4-c72a-11e3-8ef5-f3b521c818d8 2014-04-18 14:02:10 3767 [Note] WSREP: evs::proto(eb5e6359-c72a-11e3-bac7-db36affbd3b8, GATHER, view_id(REG,eb5e6359-c72a-11e3-ba c7-db36affbd3b8,2)) suspecting node: f9449da4-c72a-11e3-8ef5-f3b521c818d8 -- second node error log 2014-04-18 13:55:29 3492 [Note] /usr/sbin/mysqld: ready for connections. Version: '5.6.15-63.0-log' socket: '/var/run/mysqld/mysqld.sock' port: 3306 Percona XtraDB Cluster (GPL), Release 25.5, wsrep_25.5.r4061 2014-04-18 13:55:29 3492 [Note] WSREP: 1.0 (galera2): State transfer from 0.0 (galera1) complete. 2014-04-18 13:55:29 3492 [Note] WSREP: Shifting JOINER -> JOINED (TO: 14) 2014-04-18 13:55:29 3492 [Note] WSREP: Member 1 (galera2) synced with group. 2014-04-18 13:55:29 3492 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 14) 2014-04-18 13:55:29 3492 [Note] WSREP: Synchronized with group, ready for connections 2014-04-18 13:55:29 3492 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification. 19:02:04 UTC - mysqld got signal 11 ; This could be because you hit a bug. It is also possible that this binary or one of the libraries it was linked against is corrupt, improperly built, or misconfigured. This error can also be caused by malfunctioning hardware. We will try our best to scrape up some info that will hopefully help diagnose the problem, but since we have already crashed, something is definitely wrong and this may fail. Please help us make Percona XtraDB Cluster better by reporting any bugs at https://bugs.launchpad.net/percona-xtradb-cluster key_buffer_size=33554432 read_buffer_size=131072 max_used_connections=3 max_threads=100001 thread_count=2 connection_count=2 It is possible that mysqld could use up to key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 38435430 K bytes of memory Hope that's ok; if not, decrease some variables in the equation. Thread pointer: 0x7f14ec000990 Attempting backtrace. You can use the following information to find out where mysqld died. If you see no messages after this, something went terribly wrong... stack_bottom = 7f14f0df2a60 thread_stack 0x30000 /usr/sbin/mysqld(my_print_stacktrace+0x2e)[0x8ea56e] /usr/sbin/mysqld(handle_fatal_signal+0x4a4)[0x68d2f4] /lib/x86_64-linux-gnu/libpthread.so.0(+0xfcb0)[0x7f179310fcb0] /usr/sbin/mysqld(_ZN14Relay_log_info15cleanup_contextEP3THDb+0x34)[0x8c6c54] /usr/sbin/mysqld[0x87d7ef] /usr/sbin/mysqld(_ZN14Rows_log_event14do_apply_eventEPK14Relay_log_info+0xa6c)[0x88c69c] /usr/sbin/mysqld(_ZN9Log_event11apply_eventEP14Relay_log_info+0x72)[0x883e22] /usr/sbin/mysqld(_Z14wsrep_apply_cbPvPKvmjPK14wsrep_trx_meta+0x621)[0x5d7a91] /usr/lib/libgalera_smm.so(_ZNK6galera9TrxHandle5applyEPvPF15wsrep_cb_statusS1_PKvmjPK14wsrep_trx_metaERS6_+0xd2)[0x7f17740d64e2] /usr/lib/libgalera_smm.so(+0x1c6bbe)[0x7f1774114bbe] /usr/lib/libgalera_smm.so(_ZN6galera13ReplicatorSMM9apply_trxEPvPNS_9TrxHandleE+0xdb)[0x7f1774118f6b] /usr/lib/libgalera_smm.so(_ZN6galera13ReplicatorSMM11process_trxEPvPNS_9TrxHandleE+0x40)[0x7f17741197a0] /usr/lib/libgalera_smm.so(_ZN6galera15GcsActionSource8dispatchEPvRK10gcs_actionRb+0x3b4)[0x7f17740f4704] /usr/lib/libgalera_smm.so(_ZN6galera15GcsActionSource7processEPvRb+0x5e)[0x7f17740f52ce] /usr/lib/libgalera_smm.so(_ZN6galera13ReplicatorSMM10async_recvEPv+0x78)[0x7f177411aff8] /usr/lib/libgalera_smm.so(galera_recv+0x1e)[0x7f177412d86e] /usr/sbin/mysqld[0x5d8561] /usr/sbin/mysqld(start_wsrep_THD+0x3af)[0x5c0edf] /lib/x86_64-linux-gnu/libpthread.so.0(+0x7e9a)[0x7f1793107e9a] /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f17926223fd] Trying to get some variables. Some pointers may be invalid and cause the dump to abort. Query (7f14ec016da0): is an invalid pointer Connection ID (thread ID): 1 Status: NOT_KILLED You may download the Percona XtraDB Cluster operations manual by visiting http://www.percona.com/software/percona-xtradb-cluster/. You may find information in the manual which will help you identify the cause of the crash. 140418 14:02:04 mysqld_safe Number of processes running now: 0 140418 14:02:04 mysqld_safe WSREP: not restarting wsrep node automatically 140418 14:02:04 mysqld_safe mysqld from pid file /mysql/galera2.pid ended