This is indeed memory leak. I have repeat it with the test case provided by the customer. The analyzes is based on this code https://github.com/percona/percona-server/commit/a68f179005da7c11864f406d5c7fb0df48a60b69 . ============================<1>================================= The instruction (sp_instr_cpush) mem_root is set to persistent memory root (main_mem_root) during SP compilation bool sp_head::add_instr(THD *thd, sp_instr *instr) { ... /* Memory root of every instruction is designated for permanent transformations (optimizations) made on the parsed tree during the first execution. It points to the memory root of the entire stored procedure, as their life span is equal. */ instr->mem_root= get_persistent_mem_root(); ... } (gdb) bt #0 sp_instr_cpush::sp_instr_cpush (this=0x7fffc0059a10, ip=4, ctx=0x7fffc0047640, cursor_lex=0x7fffc0053800, cursor_query=..., cursor_idx=0) at ./sql/sp_instr.h:1209 #1 0x00000000008d3c43 in MYSQLparse (YYTHD=0x1a6ea70) at ./sql/sql_yacc.yy:3277 #2 0x00000000007e2e58 in parse_sql (thd=0x1a6ea70, parser_state=0x7ffff40a7b50, creation_ctx=0x7fffc0005890) at ./sql/sql_parse.cc:8935 #3 0x0000000000724731 in sp_compile (thd=0x1a6ea70, defstr=0x7ffff40a7de0, sql_mode=1073741824, creation_ctx=0x7fffc0005890) at ./sql/sp.cc:774 #4 0x0000000000724c54 in db_load_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, sphp=0x7ffff40a98b8, sql_mode=1073741824, params=0x7fffc0005158 "\n IN p_user_id BIGINT, \n OUT p_attributes VARCHAR(32767)", returns=0xf89290 "", body=0x7fffc00051a0 "mainpgm: BEGIN \n\nDECLARE v_continue_flag INT;\nDECLARE c_delim_pipe VARCHAR(10) DEFAULT '|';\nDECLARE c_delim_equal VARCHAR(10) DEFAULT '=';\nDECLARE v_not_found_attribute VARCH"..., chistics=..., definer=0x7fffc0005830 "sp_user@%", created=20150309002751, modified=20150309002751, creation_ctx=0x7fffc0005890) at ./sql/sp.cc:907 #5 0x0000000000724470 in db_find_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, sphp=0x7ffff40a98b8) at ./sql/sp.cc:688 #6 0x00000000007282f9 in sp_cache_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, lookup_only=false, sp=0x7ffff40a98b8) at ./sql/sp.cc:2077 #7 0x000000000072812a in sp_cache_routine (thd=0x1a6ea70, rt=0x7fffc0004d90, lookup_only=false, sp=0x7ffff40a98b8) at ./sql/sp.cc:2029 #8 0x0000000000767283 in open_and_process_routine (thd=0x1a6ea70, prelocking_ctx=0x1a70fc0, rt=0x7fffc0004d90, prelocking_strategy=0x7ffff40a9ac0, has_prelocking_list=false, ot_ctx=0x7ffff40a99d0, need_prelocking=0x7ffff40a9970, routine_modifies_data=0x7ffff40a996f) at ./sql/sql_base.cc:4481 #9 0x00000000007688ba in open_tables (thd=0x1a6ea70, start=0x7ffff40a9a40, counter=0x7ffff40a9a54, flags=0, prelocking_strategy=0x7ffff40a9ac0) at ./sql/sql_base.cc:5299 #10 0x000000000076975c in open_and_lock_tables (thd=0x1a6ea70, tables=0x0, derived=true, flags=0, prelocking_strategy=0x7ffff40a9ac0) at ./sql/sql_base.cc:5882 #11 0x000000000075b7b4 in open_and_lock_tables (thd=0x1a6ea70, tables=0x0, derived=true, flags=0) at ./sql/sql_base.h:477 #12 0x00000000007da112 in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:5001 #13 0x00000000007dec5f in mysql_parse (thd=0x1a6ea70, rawbuf=0x7fffc0004c40 "call subquery_bug_pr(1,@p_attributes_out)", length=41, parser_state=0x7ffff40ab220) at ./sql/sql_parse.cc:6922 #14 0x00000000007d0c4a in dispatch_command (command=COM_QUERY, thd=0x1a6ea70, packet=0x1b7ccc1 "call subquery_bug_pr(1,@p_attributes_out)", packet_length=41) at ./sql/sql_parse.cc:1434 #15 0x00000000007cfba0 in do_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:1051 #16 0x00000000007994c8 in do_handle_one_connection (thd_arg=0x1a6ea70) at ./sql/sql_connect.cc:1532 #17 0x0000000000798fdf in handle_one_connection (arg=0x1a6ea70) at ./sql/sql_connect.cc:1443 #18 0x00007ffff737d182 in start_thread (arg=0x7ffff40ac700) at pthread_create.c:312 #19 0x00007ffff688a47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 (gdb) bt #0 sp_head::add_instr (this=0x7fffc0045820, thd=0x1a6ea70, instr=0x7fffc0059a10) at ./sql/sp_head.cc:1638 #1 0x00000000008d3c7c in MYSQLparse (YYTHD=0x1a6ea70) at ./sql/sql_yacc.yy:3280 #2 0x00000000007e2e58 in parse_sql (thd=0x1a6ea70, parser_state=0x7ffff40a7b50, creation_ctx=0x7fffc0005890) at ./sql/sql_parse.cc:8935 #3 0x0000000000724731 in sp_compile (thd=0x1a6ea70, defstr=0x7ffff40a7de0, sql_mode=1073741824, creation_ctx=0x7fffc0005890) at ./sql/sp.cc:774 #4 0x0000000000724c54 in db_load_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, sphp=0x7ffff40a98b8, sql_mode=1073741824, params=0x7fffc0005158 "\n IN p_user_id BIGINT, \n OUT p_attributes VARCHAR(32767)", returns=0xf89290 "", body=0x7fffc00051a0 "mainpgm: BEGIN \n\nDECLARE v_continue_flag INT;\nDECLARE c_delim_pipe VARCHAR(10) DEFAULT '|';\nDECLARE c_delim_equal VARCHAR(10) DEFAULT '=';\nDECLARE v_not_found_attribute VARCH"..., chistics=..., definer=0x7fffc0005830 "sp_user@%", created=20150309002751, modified=20150309002751, creation_ctx=0x7fffc0005890) at ./sql/sp.cc:907 #5 0x0000000000724470 in db_find_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, sphp=0x7ffff40a98b8) at ./sql/sp.cc:688 #6 0x00000000007282f9 in sp_cache_routine (thd=0x1a6ea70, type=SP_TYPE_PROCEDURE, name=0x7ffff40a9690, lookup_only=false, sp=0x7ffff40a98b8) at ./sql/sp.cc:2077 #7 0x000000000072812a in sp_cache_routine (thd=0x1a6ea70, rt=0x7fffc0004d90, lookup_only=false, sp=0x7ffff40a98b8) at ./sql/sp.cc:2029 #8 0x0000000000767283 in open_and_process_routine (thd=0x1a6ea70, prelocking_ctx=0x1a70fc0, rt=0x7fffc0004d90, prelocking_strategy=0x7ffff40a9ac0, has_prelocking_list=false, ot_ctx=0x7ffff40a99d0, need_prelocking=0x7ffff40a9970, routine_modifies_data=0x7ffff40a996f) at ./sql/sql_base.cc:4481 #9 0x00000000007688ba in open_tables (thd=0x1a6ea70, start=0x7ffff40a9a40, counter=0x7ffff40a9a54, flags=0, prelocking_strategy=0x7ffff40a9ac0) at ./sql/sql_base.cc:5299 #10 0x000000000076975c in open_and_lock_tables (thd=0x1a6ea70, tables=0x0, derived=true, flags=0, prelocking_strategy=0x7ffff40a9ac0) at ./sql/sql_base.cc:5882 #11 0x000000000075b7b4 in open_and_lock_tables (thd=0x1a6ea70, tables=0x0, derived=true, flags=0) at ./sql/sql_base.h:477 #12 0x00000000007da112 in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:5001 #13 0x00000000007dec5f in mysql_parse (thd=0x1a6ea70, rawbuf=0x7fffc0004c40 "call subquery_bug_pr(1,@p_attributes_out)", length=41, parser_state=0x7ffff40ab220) at ./sql/sql_parse.cc:6922 #14 0x00000000007d0c4a in dispatch_command (command=COM_QUERY, thd=0x1a6ea70, packet=0x1b7ccc1 "call subquery_bug_pr(1,@p_attributes_out)", packet_length=41) at ./sql/sql_parse.cc:1434 #15 0x00000000007cfba0 in do_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:1051 #16 0x00000000007994c8 in do_handle_one_connection (thd_arg=0x1a6ea70) at ./sql/sql_connect.cc:1532 #17 0x0000000000798fdf in handle_one_connection (arg=0x1a6ea70) at ./sql/sql_connect.cc:1443 #18 0x00007ffff737d182 in start_thread (arg=0x7ffff40ac700) at pthread_create.c:312 #19 0x00007ffff688a47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ============================<2>================================= sp_cursor stores sp_instr_cpush Breakpoint 5, sp_cursor::sp_cursor (this=0x7fffc003acc0, i=0x7fffc0059a10) at ./sql/sp_rcontext.h:458 458 m_push_instr(i) (gdb) bt #0 sp_cursor::sp_cursor (this=0x7fffc003acc0, i=0x7fffc0059a10) at ./sql/sp_rcontext.h:458 #1 0x0000000000733cc9 in sp_rcontext::push_cursor (this=0x7fffc0072748, i=0x7fffc0059a10) at ./sql/sp_rcontext.cc:172 #2 0x000000000098e389 in sp_instr_cpush::execute (this=0x7fffc0059a10, thd=0x1a6ea70, nextp=0x7ffff40a9748) at ./sql/sp_instr.cc:1493 #3 0x000000000072c282 in sp_head::execute (this=0x7fffc0045820, thd=0x1a6ea70, merge_da_on_success=true) at ./sql/sp_head.cc:645 #4 0x000000000072e0c9 in sp_head::execute_procedure (this=0x7fffc0045820, thd=0x1a6ea70, args=0x1a71c30) at ./sql/sp_head.cc:1315 #5 0x00000000007da32b in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:5069 #6 0x00000000007dec5f in mysql_parse (thd=0x1a6ea70, rawbuf=0x7fffc0004c40 "call subquery_bug_pr(1,@p_attributes_out)", length=41, parser_state=0x7ffff40ab220) at ./sql/sql_parse.cc:6922 #7 0x00000000007d0c4a in dispatch_command (command=COM_QUERY, thd=0x1a6ea70, packet=0x1b7ccc1 "call subquery_bug_pr(1,@p_attributes_out)", packet_length=41) at ./sql/sql_parse.cc:1434 #8 0x00000000007cfba0 in do_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:1051 #9 0x00000000007994c8 in do_handle_one_connection (thd_arg=0x1a6ea70) at ./sql/sql_connect.cc:1532 #10 0x0000000000798fdf in handle_one_connection (arg=0x1a6ea70) at ./sql/sql_connect.cc:1443 #11 0x00007ffff737d182 in start_thread (arg=0x7ffff40ac700) at pthread_create.c:312 #12 0x00007ffff688a47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ============================<3>================================= thd->stmt_arena is set to the query arena of SP instruction bool sp_head::execute(THD *thd, bool merge_da_on_success) { ... /* We need to reset start_time to allow for time to flow inside a stored procedure. This is only done for SP since time is suppose to be constant during execution of triggers and functions. */ reset_start_time_for_sp(thd); /* We have to set thd->stmt_arena before executing the instruction to store in the instruction free_list all new items, created during the first execution (for example expanding of '*' or the items made during other permanent subquery transformations). */ thd->stmt_arena= i; /* Will write this SP statement into binlog separately. TODO: consider changing the condition to "not inside event union". */ if (thd->locked_tables_mode <= LTM_LOCK_TABLES) thd->user_var_events_alloc= thd->mem_root; err_status= i->execute(thd, &ip); if (i->free_list) cleanup_items(i->free_list); ... } (gdb) bt #0 sp_head::execute (this=0x7fffd0045810, thd=0x1a6ea70, merge_da_on_success=true) at ./sql/sp_head.cc:642 #1 0x000000000072e0c9 in sp_head::execute_procedure (this=0x7fffd0045810, thd=0x1a6ea70, args=0x1a71c30) at ./sql/sp_head.cc:1315 #2 0x00000000007da32b in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:5069 #3 0x00000000007dec5f in mysql_parse (thd=0x1a6ea70, rawbuf=0x7fffd0004c40 "call subquery_bug_pr(1,@p_attributes_out)", length=41, parser_state=0x7ffff40ab220) at ./sql/sql_parse.cc:6922 #4 0x00000000007d0c4a in dispatch_command (command=COM_QUERY, thd=0x1a6ea70, packet=0x1b7ccc1 "call subquery_bug_pr(1,@p_attributes_out)", packet_length=41) at ./sql/sql_parse.cc:1434 #5 0x00000000007cfba0 in do_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:1051 #6 0x00000000007994c8 in do_handle_one_connection (thd_arg=0x1a6ea70) at ./sql/sql_connect.cc:1532 #7 0x0000000000798fdf in handle_one_connection (arg=0x1a6ea70) at ./sql/sql_connect.cc:1443 #8 0x00007ffff737d182 in start_thread (arg=0x7ffff40ac700) at pthread_create.c:312 #9 0x00007ffff688a47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ============================<4>================================= sp_inst_copen is executed, thd->stmt_arena is set to sp_instr_cpush in bool sp_instr_copen::execute(THD *thd, uint *nextp) { *nextp= get_ip() + 1; // Get the cursor pointer. sp_cursor *c= thd->sp_runtime_ctx->get_cursor(m_cursor_idx); if (!c) return true; // Retrieve sp_instr_cpush instance. sp_instr_cpush *push_instr= c->get_push_instr(); // Switch Statement Arena to the sp_instr_cpush object. It contains the // free_list of the query, so new items (if any) are stored in the right // free_list, and we can cleanup after each open. Query_arena *stmt_arena_saved= thd->stmt_arena; thd->stmt_arena= push_instr; // Switch to the cursor's lex and execute sp_instr_cpush::exec_core(). // sp_instr_cpush::exec_core() is *not* executed during // sp_instr_cpush::execute(). sp_instr_cpush::exec_core() is intended to be // executed on cursor opening. bool rc= push_instr->validate_lex_and_execute_core(thd, nextp, false); // Cleanup the query's items. if (push_instr->free_list) cleanup_items(push_instr->free_list); // Restore Statement Arena. thd->stmt_arena= stmt_arena_saved; return rc; } 6178 entry= new (thd->stmt_arena->mem_root) Derived_key(); (gdb) bt #0 add_derived_key (derived_key_list=..., field=0x7fffc008ec70, ref_by_tbl=0) at ./sql/table.cc:6178 #1 0x00000000008a5819 in TABLE_LIST::update_derived_keys (this=0x7fffc0057f58, field=0x7fffc008ec70, values=0x7ffff40a7240, num_values=1) at ./sql/table.cc:6247 #2 0x00000000009bf440 in add_key_field (key_fields=0x7ffff40a7430, and_level=0, cond=0x7fffc0059408, field=0x7fffc008ec70, eq_func=true, value=0x7ffff40a7240, num_values=1, usable_tables=18446744073709551613, sargables=0x7ffff40a7558) at ./sql/sql_optimizer.cc:4907 #3 0x00000000009bfb08 in add_key_equal_fields (key_fields=0x7ffff40a7430, and_level=0, cond=0x7fffc0059408, field_item=0x7fffc0059300, eq_func=true, val=0x7ffff40a7240, num_values=1, usable_tables=18446744073709551613, sargables=0x7ffff40a7558) at ./sql/sql_optimizer.cc:5088 #4 0x00000000009c0875 in add_key_fields (join=0x7fffc0086f68, key_fields=0x7ffff40a7430, and_level=0x7ffff40a741c, cond=0x7fffc0059408, usable_tables=18446744073709551613, sargables=0x7ffff40a7558) at ./sql/sql_optimizer.cc:5315 #5 0x00000000009bfd6e in add_key_fields (join=0x7fffc0086f68, key_fields=0x7ffff40a7430, and_level=0x7ffff40a741c, cond=0x7fffc00873a8, usable_tables=18446744073709551613, sargables=0x7ffff40a7558) at ./sql/sql_optimizer.cc:5147 #6 0x00000000009c1fd4 in update_ref_and_keys (thd=0x1a6ea70, keyuse=0x7fffc0087190, join_tab=0x7fffc0096808, tables=2, cond=0x7fffc00873a8, cond_equal=0x7fffc0087498, normal_tables=18446744073709551613, select_lex=0x7fffc0053f18, sargables=0x7ffff40a7558) at ./sql/sql_optimizer.cc:5880 #7 0x00000000009bb579 in make_join_statistics (join=0x7fffc0086f68, tables_arg=0x7fffc004eac0, conds=0x7fffc00873a8, keyuse_array=0x7fffc0087190, first_optimization=true) at ./sql/sql_optimizer.cc:3349 #8 0x00000000009b398d in JOIN::optimize (this=0x7fffc0086f68) at ./sql/sql_optimizer.cc:382 #9 0x00000000008059a7 in mysql_execute_select (thd=0x1a6ea70, select_lex=0x7fffc0053f18, free_join=true) at ./sql/sql_select.cc:1086 #10 0x0000000000805cd6 in mysql_select (thd=0x1a6ea70, tables=0x7fffc004eac0, wild_num=0, fields=..., conds=0x7fffc00594e0, order=0x7fffc00540e0, group=0x7fffc0054018, having=0x0, select_options=2147749633, result=0x7fffc0085e40, unit=0x7fffc00538d0, select_lex=0x7fffc0053f18) at ./sql/sql_select.cc:1221 #11 0x0000000000803c4a in handle_select (thd=0x1a6ea70, result=0x7fffc0085e40, setup_tables_done_option=0) at ./sql/sql_select.cc:110 #12 0x00000000007dbf55 in execute_sqlcom_select (thd=0x1a6ea70, all_tables=0x7fffc004eac0) at ./sql/sql_parse.cc:5634 #13 0x00000000007d44ab in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:2989 #14 0x000000000079995d in mysql_open_cursor (thd=0x1a6ea70, result=0x7fffc003acc8, pcursor=0x7fffc003acf8) at ./sql/sql_cursor.cc:118 #15 0x0000000000734816 in sp_cursor::open (this=0x7fffc003acc0, thd=0x1a6ea70) at ./sql/sp_rcontext.cc:493 #16 0x000000000098e3df in sp_instr_cpush::exec_core (this=0x7fffc0059a10, thd=0x1a6ea70, nextp=0x7ffff40a9748) at ./sql/sp_instr.cc:1504 #17 0x000000000098be15 in sp_lex_instr::reset_lex_and_exec_core (this=0x7fffc0059a10, thd=0x1a6ea70, nextp=0x7ffff40a9748, open_tables=false) at ./sql/sp_instr.cc:391 #18 0x000000000098c796 in sp_lex_instr::validate_lex_and_execute_core (this=0x7fffc0059a10, thd=0x1a6ea70, nextp=0x7ffff40a9748, open_tables=false) at ./sql/sp_instr.cc:639 #19 0x000000000098e656 in sp_instr_copen::execute (this=0x7fffc005f2c8, thd=0x1a6ea70, nextp=0x7ffff40a9748) at ./sql/sp_instr.cc:1587 #20 0x000000000072c282 in sp_head::execute (this=0x7fffc0045820, thd=0x1a6ea70, merge_da_on_success=true) at ./sql/sp_head.cc:645 #21 0x000000000072e0c9 in sp_head::execute_procedure (this=0x7fffc0045820, thd=0x1a6ea70, args=0x1a71c30) at ./sql/sp_head.cc:1315 #22 0x00000000007da32b in mysql_execute_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:5069 #23 0x00000000007dec5f in mysql_parse (thd=0x1a6ea70, rawbuf=0x7fffc0004c40 "call subquery_bug_pr(1,@p_attributes_out)", length=41, parser_state=0x7ffff40ab220) at ./sql/sql_parse.cc:6922 #24 0x00000000007d0c4a in dispatch_command (command=COM_QUERY, thd=0x1a6ea70, packet=0x1b7ccc1 "call subquery_bug_pr(1,@p_attributes_out)", packet_length=41) at ./sql/sql_parse.cc:1434 #25 0x00000000007cfba0 in do_command (thd=0x1a6ea70) at ./sql/sql_parse.cc:1051 #26 0x00000000007994c8 in do_handle_one_connection (thd_arg=0x1a6ea70) at ./sql/sql_connect.cc:1532 #27 0x0000000000798fdf in handle_one_connection (arg=0x1a6ea70) at ./sql/sql_connect.cc:1443 #28 0x00007ffff737d182 in start_thread (arg=0x7ffff40ac700) at pthread_create.c:312 #29 0x00007ffff688a47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ============================================================= This string entry= new (thd->stmt_arena->mem_root) Derived_key(); allocates memory on persistent memory root (main_mem_root) the pointer on which is stored in sp_instr_cpush during SP body compilation(see (1)) and thd->stmt_arena is set equal to sp_instr_cpush in sp_instr_copen::execute() (see (4) and (2) ).