mosquitto crashes when lots of clients are connected

[Roger Light]

Reported on mqtt-users mailing list

I'm getting the following crash from mosquitto 0.9.1 running on Ubuntu. It was not happening at all with a lower number of outstanding connections, but it appears to be happening consistently now with more outstanding connections (~2500 open).

Any ideas?

*** glibc detected *** mosquitto: realloc(): invalid next size: 0x00000000015d7b90 ***
======= Backtrace: =========
/lib/libc.so.6(+0x775b6)[0x7f1fd68565b6]
/lib/libc.so.6(+0x7dd96)[0x7f1fd685cd96]
/lib/libc.so.6(realloc+0xf0)[0x7f1fd685d0b0]
mosquitto[0x40638f]
mosquitto[0x4023ce]
mosquitto[0x402c72]
/lib/libc.so.6(__libc_start_main+0xfd)[0x7f1fd67fdc4d]
mosquitto[0x401c69]
======= Memory map: ========
00400000-00410000 r-xp 00000000 ca:01 1051396 /usr/local/sbin/mosquitto
0060f000-00610000 r--p 0000f000 ca:01 1051396 /usr/local/sbin/mosquitto
00610000-00611000 rw-p 00010000 ca:01 1051396 /usr/local/sbin/mosquitto
014e0000-015eb000 rw-p 00000000 00:00 0 [heap]
7f1fd0000000-7f1fd0021000 rw-p 00000000 00:00 0
7f1fd0021000-7f1fd4000000 ---p 00000000 00:00 0
7f1fd596c000-7f1fd5982000 r-xp 00000000 ca:01 458959 /lib/libgcc_s.so.1
7f1fd5982000-7f1fd5b81000 ---p 00016000 ca:01 458959 /lib/libgcc_s.so.1
7f1fd5b81000-7f1fd5b82000 r--p 00015000 ca:01 458959 /lib/libgcc_s.so.1
7f1fd5b82000-7f1fd5b83000 rw-p 00016000 ca:01 458959 /lib/libgcc_s.so.1
7f1fd5b83000-7f1fd5b8f000 r-xp 00000000 ca:01 458976 /lib/libnss_files-2.11.1.so
7f1fd5b8f000-7f1fd5d8e000 ---p 0000c000 ca:01 458976 /lib/libnss_files-2.11.1.so
7f1fd5d8e000-7f1fd5d8f000 r--p 0000b000 ca:01 458976 /lib/libnss_files-2.11.1.so
7f1fd5d8f000-7f1fd5d90000 rw-p 0000c000 ca:01 458976 /lib/libnss_files-2.11.1.so
7f1fd5d90000-7f1fd5d9a000 r-xp 00000000 ca:01 458978 /lib/libnss_nis-2.11.1.so
7f1fd5d9a000-7f1fd5f99000 ---p 0000a000 ca:01 458978 /lib/libnss_nis-2.11.1.so
7f1fd5f99000-7f1fd5f9a000 r--p 00009000 ca:01 458978 /lib/libnss_nis-2.11.1.so
7f1fd5f9a000-7f1fd5f9b000 rw-p 0000a000 ca:01 458978 /lib/libnss_nis-2.11.1.so
7f1fd5f9b000-7f1fd5fb2000 r-xp 00000000 ca:01 458973 /lib/libnsl-2.11.1.so
7f1fd5fb2000-7f1fd61b1000 ---p 00017000 ca:01 458973 /lib/libnsl-2.11.1.so
7f1fd61b1000-7f1fd61b2000 r--p 00016000 ca:01 458973 /lib/libnsl-2.11.1.so
7f1fd61b2000-7f1fd61b3000 rw-p 00017000 ca:01 458973 /lib/libnsl-2.11.1.so
7f1fd61b3000-7f1fd61b5000 rw-p 00000000 00:00 0
7f1fd61b5000-7f1fd61bd000 r-xp 00000000 ca:01 458974 /lib/libnss_compat-2.11.1.so
7f1fd61bd000-7f1fd63bc000 ---p 00008000 ca:01 458974 /lib/libnss_compat-2.11.1.so
7f1fd63bc000-7f1fd63bd000 r--p 00007000 ca:01 458974 /lib/libnss_compat-2.11.1.so
7f1fd63bd000-7f1fd63be000 rw-p 00008000 ca:01 458974 /lib/libnss_compat-2.11.1.so
7f1fd63be000-7f1fd63c0000 r-xp 00000000 ca:01 458950 /lib/libdl-2.11.1.so
7f1fd63c0000-7f1fd65c0000 ---p 00002000 ca:01 458950 /lib/libdl-2.11.1.so
7f1fd65c0000-7f1fd65c1000 r--p 00002000 ca:01 458950 /lib/libdl-2.11.1.so
7f1fd65c1000-7f1fd65c2000 rw-p 00003000 ca:01 458950 /lib/libdl-2.11.1.so
7f1fd65c2000-7f1fd65da000 r-xp 00000000 ca:01 458992 /lib/libpthread-2.11.1.so
7f1fd65da000-7f1fd67d9000 ---p 00018000 ca:01 458992 /lib/libpthread-2.11.1.so
7f1fd67d9000-7f1fd67da000 r--p 00017000 ca:01 458992 /lib/libpthread-2.11.1.so
7f1fd67da000-7f1fd67db000 rw-p 00018000 ca:01 458992 /lib/libpthread-2.11.1.so
7f1fd67db000-7f1fd67df000 rw-p 00000000 00:00 0
7f1fd67df000-7f1fd6959000 r-xp 00000000 ca:01 458942 /lib/libc-2.11.1.so
7f1fd6959000-7f1fd6b58000 ---p 0017a000 ca:01 458942 /lib/libc-2.11.1.so
7f1fd6b58000-7f1fd6b5c000 r--p 00179000 ca:01 458942 /lib/libc-2.11.1.so
7f1fd6b5c000-7f1fd6b5d000 rw-p 0017d000 ca:01 458942 /lib/libc-2.11.1.so
7f1fd6b5d000-7f1fd6b62000 rw-p 00000000 00:00 0
7f1fd6b62000-7f1fd6bec000 r-xp 00000000 ca:01 1017625 /usr/lib/libsqlite3.so.0.8.6
7f1fd6bec000-7f1fd6deb000 ---p 0008a000 ca:01 1017625 /usr/lib/libsqlite3.so.0.8.6
7f1fd6deb000-7f1fd6ded000 r--p 00089000 ca:01 1017625 /usr/lib/libsqlite3.so.0.8.6
7f1fd6ded000-7f1fd6def000 rw-p 0008b000 ca:01 1017625 /usr/lib/libsqlite3.so.0.8.6
7f1fd6def000-7f1fd6e0f000 r-xp 00000000 ca:01 458932 /lib/ld-2.11.1.so
7f1fd7004000-7f1fd7008000 rw-p 00000000 00:00 0
7f1fd700c000-7f1fd700e000 rw-p 00000000 00:00 0
7f1fd700e000-7f1fd700f000 r--p 0001f000 ca:01 458932 /lib/ld-2.11.1.so
7f1fd700f000-7f1fd7010000 rw-p 00020000 ca:01 458932 /lib/ld-2.11.1.so
7f1fd7010000-7f1fd7011000 rw-p 00000000 00:00 0
7fffde5e9000-7fffde60a000 rw-p 00000000 00:00 0 [stack]
7fffde754000-7fffde755000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]

[Roger Light (roger.light)]

Stack trace provided.

Program terminated with signal 6, Aborted.
#0 0x00007f13d034fa75 in raise () from /lib/libc.so.6
(gdb) bt
#0 0x00007f13d034fa75 in raise () from /lib/libc.so.6
#1 0x00007f13d03535c0 in abort () from /lib/libc.so.6
#2 0x00007f13d03894fb in ?? () from /lib/libc.so.6
#3 0x00007f13d03935b6 in ?? () from /lib/libc.so.6
#4 0x00007f13d0399d96 in ?? () from /lib/libc.so.6
#5 0x00007f13d039a0b0 in realloc () from /lib/libc.so.6
#6 0x000000000040638f in _mosquitto_realloc (ptr=0x1e51980, size=8088) at ../lib/memory_mosq.c:97
#7 0x00000000004023ce in loop (config=<value optimized out>, listensock=0x1de2620, listensock_count=<value optimized out>, listener_max=<value optimized out>) at mosquitto.c:149
#8 0x0000000000402c72 in main (argc=<value optimized out>, argv=0x2) at mosquitto.c:470

[Roger Light (roger.light)]

I haven't been able to reproduce this crash. I've tried a number of different approaches that consist of having up to 4000 mosquitto_sub and mosquitto_pub instances connected to a single mosquitto broker at once. Even running mosquitto through valgrind when doing the same tests produces no memory errors.

Any details on how to reproduce the crash would be appreciated.