stack overflow on sasl authentication failure

[sinny]

libmemcached version: 1.0.3, 1.0.4

the following code (extracted, simplified):
-----------------------------------------------------------------------------
        if ( sasl_client_init(NULL) != SASL_OK )
        {
                SINNY_ERR("error : sasl_client_init() failed");
                throw 1;
        }

        {
                if ( NULL == (memc_ = memcached_create(NULL)) )
                {
                        SINNY_ERR("error : memcached_create() -> NULL");
                        throw 1;
                }

                memcached_return_t rc;

                rc = memcached_set_sasl_auth_data(memc_, username.c_str(), password.c_str());
                if ( rc != MEMCACHED_SUCCESS )
                {
                        SINNY_ERR("error : memcached_set_sasl_auth_data() -> %s (%d)", z_decode_rc(rc, MEMC_ERRNO(memc_)).c_str(), rc);
                        throw 1;
                }

                memcached_server_st *srv = memcached_servers_parse(servers.c_str());
                rc = memcached_server_push(memc_, srv);

...

-----------------------------------------------------------------------------

results in segfault with the following backtrace:
-----------------------------------------------------------------------------
(gdb) bt
#0 0x001553ab in vfprintf () from /lib/tls/libc.so.6
#1 0x00178d24 in vsnprintf () from /lib/tls/libc.so.6
#2 0x0015e0a4 in snprintf () from /lib/tls/libc.so.6
#3 0x080d2240 in memcached_set_error ()
#4 0x080d8900 in binary_read_one_response ()
#5 0x080d8f36 in _read_one_response ()
#6 0x080d9138 in memcached_response ()
#7 0x080d90ad in memcached_response ()
#8 0x080dba67 in memcached_sasl_authenticate_connection ()
#9 0x080d0e49 in _memcached_connect ()
#10 0x080d0ef9 in memcached_connect_try ()
#11 0x080d95d8 in __server_create_with ()

#12 0x080d9774 in memcached_server_clone ()
#13 0x080d7f38 in memcached_mark_server_for_timeout ()
#14 0x080d7da8 in memcached_quit_server ()
#15 0x080d63e9 in memcached_io_reset ()
#16 0x080d8fa8 in _read_one_response ()
#17 0x080d9138 in memcached_response ()
#18 0x080d90ad in memcached_response ()
#19 0x080dba67 in memcached_sasl_authenticate_connection ()
#20 0x080d0e49 in _memcached_connect ()
#21 0x080d0ef9 in memcached_connect_try ()
#22 0x080d95d8 in __server_create_with ()

#23 0x080d9774 in memcached_server_clone ()
#24 0x080d7f38 in memcached_mark_server_for_timeout ()
#25 0x080d7da8 in memcached_quit_server ()
#26 0x080d63e9 in memcached_io_reset ()
#27 0x080d8fa8 in _read_one_response ()
#28 0x080d9138 in memcached_response ()
#29 0x080d90ad in memcached_response ()
#30 0x080dba67 in memcached_sasl_authenticate_connection ()
#31 0x080d0e49 in _memcached_connect ()
#32 0x080d0ef9 in memcached_connect_try ()
#33 0x080d95d8 in __server_create_with ()

...

#9252 0x080d9774 in memcached_server_clone ()
#9253 0x080d7f38 in memcached_mark_server_for_timeout ()
#9254 0x080d7da8 in memcached_quit_server ()
#9255 0x080d63e9 in memcached_io_reset ()
#9256 0x080d8fa8 in _read_one_response ()
#9257 0x080d9138 in memcached_response ()
#9258 0x080d90ad in memcached_response ()
#9259 0x080dba67 in memcached_sasl_authenticate_connection ()
#9260 0x080d0e49 in _memcached_connect ()
#9261 0x080d0ef9 in memcached_connect_try ()
#9262 0x080d95d8 in __server_create_with ()

#9263 0x080d9774 in memcached_server_clone ()
#9264 0x080d7f38 in memcached_mark_server_for_timeout ()
#9265 0x080d7da8 in memcached_quit_server ()
#9266 0x080d63e9 in memcached_io_reset ()
#9267 0x080d8fa8 in _read_one_response ()
#9268 0x080d9138 in memcached_response ()
#9269 0x080d90ad in memcached_response ()
#9270 0x080dba67 in memcached_sasl_authenticate_connection ()
#9271 0x080d0e49 in _memcached_connect ()
#9272 0x080d0ef9 in memcached_connect_try ()
#9273 0x080d95d8 in __server_create_with ()

#9274 0x080d523a in memcached_server_push ()
#9275 0x0805112a in MemcachedPool (this=0x9133780, initial=5, max=100, servers=@0x912a04c, username=@0xb67de9f0, password=@0x912a050, mode=152296224, to_conn=5000, to_send=5000000, to_recv=5000000) at backend/memcached.cpp:317
#9276 0x0804f9e9 in Box::Backend::Memcached::z_get (this=0x912a010, pool_id=@0xb67de9f0) at backend/memcached.cpp:236
#9277 0x08074693 in Box::TPTFHandler::thread_tim_coll (this=0x9124518, context=@0x91335ac, th=0x9133570) at backend/interface.h:35
#9278 0x0807c16c in Box::Timer::CollectorThread::threadProc (this=0x9133570) at timer.cpp:686
#9279 0x080a7c1d in P90::Thread::threadProcEntry () at /usr/include/c++/3.2.3/bits/stl_iterator.h:599
#9280 0x00c2cdd8 in start_thread () from /lib/tls/libpthread.so.0
#9281 0x001edfca in clone () from /lib/tls/libc.so.6
(gdb)
-----------------------------------------------------------------------------

this happens when sasl authentication fails, e.g.:
 - connecting to membase/couchbase with bucket/password not registered in backend
 - connecting to text-mode memcached (with libmemcached operating in binary/sasl mode)
 - etc

the same code worked ok with version 0.47.

is there any other info that could help?

[sinny (sinnydono)]

note: in provided code sample, "memcached_server_st *srv = memcached_servers_parse(servers.c_str());" line: <servers> string contains single ip/hostname.

[Brian Aker (brianaker)]

Have you tried creating a string and using memcached() to create your structure?

[sinny (sinnydono)]

more concise scenario (libmemcached 1.0.4):
 - start memcached in text mode on host <addr> ( memcached -d -p 11211 -u memcached -m 64 -c 1024 -P /var/run/memcached/memcached.pid )
 - compile attached file ( make test CXXFLAGS=-g LDLIBS=-lmemcached )
 - execute compiled file ( ./test <addr>:11211 TEST test )

in my case this leads to the following:
-------------------------------------------------------
[root@ws home]# ./test 192.168.65.3:11211 TEST test

USAGE : ./test server_string sasl_user sasl_password

Segmentation fault
[root@ws home]#
-------------------------------------------------------

with the following backtrace (tail only):
-------------------------------------------------------
...
#6109 0xf7fc5f16 in memcached_server_clone(memcached_server_st*, memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6110 0xf7fc3ef0 in memcached_quit_server(memcached_server_st*, bool) () from /usr/lib/libmemcached.so.9
#6111 0xf7fc1125 in memcached_io_reset(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6112 0xf7fc537a in ?? () from /usr/lib/libmemcached.so.9
#6113 0xf7fc5477 in memcached_response(memcached_server_st*, char*, unsigned int, memcached_result_st*, unsigned long long&) () from /usr/lib/libmemcached.so.9
#6114 0xf7fc5639 in memcached_response(memcached_server_st*, char*, unsigned int, memcached_result_st*) () from /usr/lib/libmemcached.so.9
#6115 0xf7fcba9c in memcached_sasl_authenticate_connection(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6116 0xf7fbaa2f in ?? () from /usr/lib/libmemcached.so.9
#6117 0xf7fbaf19 in memcached_connect_try(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6118 0xf7fc5de0 in __server_create_with(memcached_st*, memcached_server_st*, memcached_string_t const&, unsigned short, unsigned int, memcached_connection_t) () from /usr/lib/libmemcached.so.9
#6119 0xf7fc5f16 in memcached_server_clone(memcached_server_st*, memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6120 0xf7fc3ef0 in memcached_quit_server(memcached_server_st*, bool) () from /usr/lib/libmemcached.so.9
#6121 0xf7fc1125 in memcached_io_reset(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6122 0xf7fc537a in ?? () from /usr/lib/libmemcached.so.9
#6123 0xf7fc5477 in memcached_response(memcached_server_st*, char*, unsigned int, memcached_result_st*, unsigned long long&) () from /usr/lib/libmemcached.so.9
#6124 0xf7fc5639 in memcached_response(memcached_server_st*, char*, unsigned int, memcached_result_st*) () from /usr/lib/libmemcached.so.9
#6125 0xf7fcba9c in memcached_sasl_authenticate_connection(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6126 0xf7fbaa2f in ?? () from /usr/lib/libmemcached.so.9
#6127 0xf7fbaf19 in memcached_connect_try(memcached_server_st*) () from /usr/lib/libmemcached.so.9
#6128 0xf7fc5de0 in __server_create_with(memcached_st*, memcached_server_st*, memcached_string_t const&, unsigned short, unsigned int, memcached_connection_t) () from /usr/lib/libmemcached.so.9
#6129 0xf7fc0fa4 in memcached_server_push () from /usr/lib/libmemcached.so.9
#6130 0x08048bef in main (argc=4, argv=0xffffdb34) at test.cpp:33
-------------------------------------------------------
...

[douyuan (douyuan)]

Got similar problem with "binary protocol" + "sasl auth with a wrong password".
The key is that memcached_io_reset on a MEMCACHED_AUTH_FAILURE-ed server will lead to infinite recursion.
I've made a quick & dirty fix. Hope this patch is useful to you.

Warning!
Since __server_create_with returns a pointer only, there is no way to distinguish between auth failure and memory allocation failure.

The following is part of my backtrace:
-------------------------
Program received signal SIGSEGV, Segmentation fault.
0x00000039fae42854 in vfprintf () from /lib64/libc.so.6
(gdb) bt
#0 0x00000039fae42854 in vfprintf () from /lib64/libc.so.6
#1 0x00000039fae63c99 in vsprintf () from /lib64/libc.so.6
#2 0x00000039fae4d678 in sprintf () from /lib64/libc.so.6
#3 0x00000039faee0c0e in inet_ntop () from /lib64/libc.so.6
#4 0x00000039faef2a99 in getnameinfo () from /lib64/libc.so.6
#5 0x00002aaaafe5017c in resolve_names (server=..., laddr=0x7fffff4028e0 "\023", laddr_length=1057, raddr=0x7fffff4024b0 " ",
    raddr_length=1057) at libmemcached/sasl.cc:80
#6 0x00002aaaafe503ec in memcached_sasl_authenticate_connection (server=0x1e2bb60) at libmemcached/sasl.cc:183
#7 0x00002aaaafe42d7c in _memcached_connect (server=0x1e2bb60, set_last_disconnected=false) at libmemcached/connect.cc:665
#8 0x00002aaaafe42f5a in memcached_connect_try (server=0x1e2bb60) at libmemcached/connect.cc:715
#9 0x00002aaaafe51222 in __server_create_with (memc=0x725d90, self=0x1e2bb60, hostname=..., port=11211, weight=1,
    type=MEMCACHED_CONNECTION_TCP) at libmemcached/server.cc:143
#10 0x00002aaaafe512df in memcached_server_clone (destination=0x0, source=0x1e26000) at libmemcached/server.cc:204
#11 0x00002aaaafe4cee8 in set_last_disconnected_host (self=0x1e26000) at ./libmemcached/server.hpp:93
#12 0x00002aaaafe4cfc6 in memcached_mark_server_for_timeout (server=0x1e26000) at ./libmemcached/server.hpp:117
#13 0x00002aaaafe4d211 in memcached_quit_server (ptr=0x1e26000, io_death=true) at libmemcached/quit.cc:131
#14 0x00002aaaafe497c8 in memcached_io_reset (ptr=0x1e26000) at libmemcached/io.cc:743
#15 0x00002aaaafe4efce in _read_one_response (instance=0x1e26000, buffer=0x0, buffer_length=0, result=0x725e30)
    at libmemcached/response.cc:814
#16 0x00002aaaafe4f188 in memcached_response (instance=0x1e26000, buffer=0x0, buffer_length=0, result=0x0) at libmemcached/response.cc:883
#17 0x00002aaaafe507d9 in memcached_sasl_authenticate_connection (server=0x1e26000) at libmemcached/sasl.cc:251
#18 0x00002aaaafe42d7c in _memcached_connect (server=0x1e26000, set_last_disconnected=false) at libmemcached/connect.cc:665
#19 0x00002aaaafe42f5a in memcached_connect_try (server=0x1e26000) at libmemcached/connect.cc:715
#20 0x00002aaaafe51222 in __server_create_with (memc=0x725d90, self=0x1e26000, hostname=..., port=11211, weight=1,
    type=MEMCACHED_CONNECTION_TCP) at libmemcached/server.cc:143
#21 0x00002aaaafe512df in memcached_server_clone (destination=0x0, source=0x1e204a0) at libmemcached/server.cc:204
#22 0x00002aaaafe4cee8 in set_last_disconnected_host (self=0x1e204a0) at ./libmemcached/server.hpp:93
#23 0x00002aaaafe4cfc6 in memcached_mark_server_for_timeout (server=0x1e204a0) at ./li...