apache2 segfault using mod_deflate

[Sylvain Filteau]

Binary package hint: apache2.2-common

On my production webserver, I started having segfault in my error log :

Jul 28 04:32:08 2009] [notice] child pid 9005 exit signal Segmentation fault (11)
Jul 28 05:30:53 2009] [notice] child pid 15156 exit signal Segmentation fault (11)
Jul 28 05:32:52 2009] [notice] child pid 15204 exit signal Segmentation fault (11)
Jul 28 05:39:18 2009] [notice] child pid 15013 exit signal Segmentation fault (11)
Jul 28 05:45:33 2009] [notice] child pid 15202 exit signal Segmentation fault (11)
[...]

Here is a gdb backtrace of a core dump :

(gdb) bt full
#0 0x00007f91e7e9bd37 in crc32 () from /usr/lib/libz.so.1
No symbol table info available.
#1 0x00007f91e5ffd204 in deflate_out_filter (f=0xaab9d0, bb=0xaa4978)
    at /build/buildd/apache2-2.2.8/modules/filters/mod_deflate.c:698
 data = 0x36b7f88 <Address 0x36b7f88 out of bounds>
 b = <value optimized out>
 len = 2523705
 e = (apr_bucket *) 0xa99f58
 r = (request_rec *) 0xaab598
 ctx = (deflate_ctx *) 0xaa4c70
 zRC = <value optimized out>
 c = (deflate_filter_config *) 0x6dfda8
#2 0x00007f91e5358bbb in ?? () from /usr/lib/apache2/modules/libphp5.so
No symbol table info available.
#3 0x0000000000437daa in ap_run_handler (r=0xaab598)
    at /build/buildd/apache2-2.2.8/server/config.c:158
 n = 3
 rv = 2523705
#4 0x000000000043b1cc in ap_invoke_handler (r=0xaab598)
    at /build/buildd/apache2-2.2.8/server/config.c:373
 handler = 0x7d19c8 "application/x-httpd-php"
 result = 0
 old_handler = 0x0
 ignore = <value optimized out>
#5 0x000000000044773a in ap_internal_redirect (new_uri=<value optimized out>,
    r=<value optimized out>)
    at /build/buildd/apache2-2.2.8/modules/http/http_request.c:477
 new = (request_rec *) 0xaab598
 access_status = 0
#6 0x00007f91e441f2d0 in handler_redirect (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/modules/mappers/mod_rewrite.c:4762
No locals.
#7 0x0000000000437daa in ap_run_handler (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/server/config.c:158
 n = 4
 rv = 2523705
#8 0x000000000043b1cc in ap_invoke_handler (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/server/config.c:373
 handler = 0x200000000 <Address 0x200000000 out of bounds>
 result = 0
 old_handler = 0x7f91e4423aab "redirect-handler"
 ignore = <value optimized out>
#9 0x00000000004478ae in ap_process_request (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/modules/http/http_request.c:258
 access_status = 0
#10 0x0000000000444ca8 in ap_process_http_connection (c=0xa95b58)
    at /build/buildd/apache2-2.2.8/modules/http/http_core.c:190
 r = (request_rec *) 0xaa1ca8
 csd = (apr_socket_t *) 0x0
#11 0x000000000043ef02 in ap_run_process_connection (c=0xa95b58)
    at /build/buildd/apache2-2.2.8/server/connection.c:43
 n = 0
 rv = 2523705
---Type <return> to continue, or q <return> to quit---
#12 0x000000000044b6a5 in child_main (child_num_arg=<value optimized out>)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:662
 current_conn = (conn_rec *) 0xa95b58
 csd = (void *) 0xa95968
 ptrans = (apr_pool_t *) 0xa958f8
 allocator = (apr_allocator_t *) 0xa937f0
 status = <value optimized out>
 i = <value optimized out>
 lr = <value optimized out>
 pollset = (apr_pollset_t *) 0xa939e8
 sbh = (ap_sb_handle_t *) 0xa939e0
 bucket_alloc = (apr_bucket_alloc_t *) 0xa99bf8
 last_poll_idx = 1
#13 0x000000000044b955 in make_child (s=0x674968, slot=7)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:759
 pid = 0
#14 0x000000000044c1e8 in ap_mpm_run (_pconf=<value optimized out>,
    plog=<value optimized out>, s=<value optimized out>)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:894
 status = 0
 pid = {pid = -1, in = 0x8485d0, out = 0x676180, err = 0x668040}
 child_slot = <value optimized out>
 exitwhy = APR_PROC_EXIT
 processed_status = <value optimized out>
 index = <value optimized out>
 remaining_children_to_start = 0
 rv = <value optimized out>
#15 0x0000000000425a44 in main (argc=3, argv=0x7ffff3a90848)
    at /build/buildd/apache2-2.2.8/server/main.c:732
 c = 0 '\0'
 configtestonly = 0
 confname = 0x44ddba "/etc/apache2/apache2.conf"
 def_server_root = 0x45296a ""
 temp_error_log = 0x0
 error = <value optimized out>
 process = (process_rec *) 0x66c238
 server_conf = (server_rec *) 0x674968
 pglobal = (apr_pool_t *) 0x66c158
 pconf = (apr_pool_t *) 0x66e168
 plog = (apr_pool_t *) 0x6a2308
 ptemp = (apr_pool_t *) 0x6761a8
 pcommands = (apr_pool_t *) 0x670178
 opt = (apr_getopt_t *) 0x670260
 rv = 0
 optarg = 0x7ffff3a90848 "8\017���\177"

--------------------------------------

A little search on google pop me out this page : http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=537665 (DSA-1834-2
) talking about a bug that looks like my issue. The reporter have done something with gdb that I copy and pasted and I thought maybe it could help with this bug report :

(gdb) select 1
(gdb) p *r
$1 = {pool = 0xaa1c38, connection = 0xa95b58, server = 0x7fb040, next = 0x0,
  prev = 0xaa1ca8, main = 0x0, the_request = 0xaa3238 "POST /siam/engin HTTP/1.1",
  assbackwards = 0, proxyreq = 0, header_only = 0, protocol = 0xaa32c0 "HTTP/1.1",
  proto_num = 1001, hostname = 0xaa3938 "[hidden-hostname]",
  request_time = 1249575009636661, status_line = 0x454fd3 "200 OK", status = 200,
  method = 0xaa3288 "POST", method_number = 2, allowed = 0, allowed_xmethods = 0x0,
  allowed_methods = 0xa9c898, sent_bodyct = 1, bytes_sent = 56682, mtime = 0,
  chunked = 1, range = 0x0, clength = 0, remaining = 0, read_length = 0,
  read_body = 0, read_chunked = 0, expecting_100 = 0, headers_in = 0xaa1f88,
  headers_out = 0xa9c130, err_headers_out = 0xaa28d0, subprocess_env = 0xa9c378,
  notes = 0xa9c6f8, content_type = 0xaa4bb0 "text/html",
  handler = 0x7d19c8 "application/x-httpd-php", content_encoding = 0x0,
  content_languages = 0x0, vlist_validator = 0xaab130 "\"44bd08b592a80\"",
  user = 0x0, ap_auth_type = 0x0, no_cache = 0, no_local_copy = 1,
  unparsed_uri = 0xaab878 "/index.php/srv/www/sygestran/production/htdocs/siam/engin", uri = 0xaab8b8 "/index.php/srv/www/sygestran/production/htdocs/siam/engin",
  filename = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
  canonical_filename = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
  path_info = 0xa9ce76 "/srv/www/sygestran/production/htdocs/siam/engin",
  args = 0x0, finfo = {pool = 0xaa1c38, valid = 7598448, protection = 1604,
    filetype = APR_REG, user = 1000, group = 1000, inode = 1426560, device = 2056,
    nlink = 1, size = 3199, csize = 8598318192, atime = 1213997387000000,
    mtime = 1213997387000000, ctime = 1213997387000000,
    fname = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
    name = 0x4384cd "I\211\004,H\213[ H\205�t5HcC\bI\213T�", filehand = 0xa9c970},
  parsed_uri = {scheme = 0x0, hostinfo = 0x0, user = 0x0, password = 0x0,
    hostname = 0x0, port_str = 0x0,
    path = 0xaab8b8 "/index.php/srv/www/sygestran/production/htdocs/siam/engin",
    query = 0x0, fragment = 0x0, hostent = 0x0, port = 0, is_initialized = 1,
    dns_looked_up = 0, dns_resolved = 0}, used_path_info = 0,
  per_dir_config = 0xa9d568, request_config = 0xa9bc08, htaccess = 0xa9e1f8,
  output_filters = 0xaa4c00, input_filters = 0xaa3958,
  proto_output_filters = 0xaa3180, proto_input_filters = 0xaa3958, eos_sent = 1}

--------------------------------------

$ lsb_release -rd
Description: Ubuntu 8.04.3 LTS
Release: 8.04

$ apt-cache policy apache2.2-common
apache2.2-common:
  Installed: 2.2.8-1ubuntu0.10
  Candidate: 2.2.8-1ubuntu0.10
  Version table:
 *** 2.2.8-1ubuntu0.10 0
        500 http://ca.archive.ubuntu.com hardy-updates/main Packages
        500 http://ca.archive.ubuntu.com hardy-security/main Packages
        100 /var/lib/dpkg/status
     2.2.8-1 0
        500 http://ca.archive.ubuntu.com hardy/main Packages

[Chuck Short (zulcss)]

Can you try the version in my ppa when its built (http://launchpad.net/~zulcss/+archive)?

Thanks
chuck

[Marc Deslauriers (mdeslaur)]

Relevant thread: http://<email address hidden>/msg44655.html

[Sylvain Filteau (cidsphere)]

@chuck I installed the apache version of your ppa :
$ apache2 -v
Server version: Apache/2.2.8 (Ubuntu)
Server built: Aug 7 2009 13:02:54

Sadly, it didn't resolved my issue

-----

@marc I tried the php script but I can't trigger the segfault with it...

-----

Since this morning I work to reproduce the problem in my dev environment with success.

Now I try to build a script that reproduce my problem but this task is hard because my application is really big. I can say that the output is a json string of 2.5M with these http headers :

HTTP/1.1 100 Continue

HTTP/1.1 200 OK
Date: Fri, 07 Aug 2009 20:06:31 GMT
Server: Apache/2.2.8 (Ubuntu) PHP/5.2.4-2ubuntu5.6 with Suhosin-Patch
X-Powered-By: PHP/5.2.4-2ubuntu5.6
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
ADAFramework-identity: some_string_important_for_me
Vary: Accept-Encoding
Content-Encoding: gzip
Transfer-Encoding: chunked
Content-Type: text/html

-----

Have a nice weekend !

[Sylvain Filteau (cidsphere)]

I wrote a small php script that triggered my problem :

<?php
echo file_get_contents('big-shuffled.json');
?>

Put this as 'x.php' in your apache document root with the file in attachment and run this command :
$ curl -is -H 'Accept-Encoding: gzip' http://localhost/x.php

On my machine, it triggered the problem.

I tried to generate a 2.5M of random stuff but it didn't work. Only this file does the problem with mod_deflate.

[Marc Deslauriers (mdeslaur)]

Thanks for the reproducer Sylvain. I could now reproduce the segfault locally.

Could you please try the updated packages in my PPA:

https://launchpad.net/~mdeslaur/+archive/ppa

If they solve the segfaults for you, I'll push out some updates.

Thanks.

[Sylvain Filteau (cidsphere)]

Looks good on my side !

Thank you very much for your help !

[Launchpad Janitor (janitor)]

This bug was fixed in the package apache2 - 2.2.12-1ubuntu2

---------------
apache2 (2.2.12-1ubuntu2) karmic; urgency=low

  * debian/patches/203_fix_legacy_ap_rputs_segfaults.dpatch:
    - Fix potential segfaults with the use of the legacy ap_rputs() etc
      interfaces, in cases where an output filter fails. This happens
      frequently after CVE-2009-1891 got fixed. (LP: #409987)

 -- Marc Deslauriers <email address hidden> Mon, 17 Aug 2009 15:38:47 -0400

[Marc Deslauriers (mdeslaur)]

Updates for current releases were just published:

http://www.ubuntu.com/usn/USN-802-2

[tiiibs (tiiibs)]

Hi,

I've the same problem. I've patched the server but the error is still here!

[Tue Aug 25 11:25:02 2009] [notice] child pid 10025 exit signal Segmentation fault (11)
[Tue Aug 25 11:25:02 2009] [notice] child pid 10026 exit signal Segmentation fault (11)
[Tue Aug 25 11:25:02 2009] [notice] child pid 10027 exit signal Segmentation fault (11)

what are tests that I can do?

[Marc Deslauriers (mdeslaur)]

tiiibs: are you sure it's the same problem? What apache2 package version are you running? What release of Ubuntu?

[Xeno (xeno22)]

Also have this issue on some servers.

Running Ubuntu 14.04 LTS:

Server version: Apache/2.4.7 (Ubuntu)
Server built: May 9 2017 16:14:10

root@server:/var/www/obs# php -v
PHP 5.5.9-1ubuntu4.21 (cli) (built: Feb 9 2017 20:54:58)
Copyright (c) 1997-2014 The PHP Group
Zend Engine v2.5.0, Copyright (c) 1998-2014 Zend Technologies
    with SourceGuardian v11.1.0, Copyright (c) 2000-2017, by SourceGuardian Ltd.
    with Zend OPcache v7.0.3, Copyright (c) 1999-2014, by Zend Technologies

root@server:/var/www/obs# apt-cache policy zlib1g
zlib1g:
  Installed: 1:1.2.8.dfsg-1ubuntu1

root@server:/var/www/obs# apt-cache policy apache2
apache2:
  Installed: 2.4.7-1ubuntu4.15

root@server:/var/www/obs# apt-cache policy php5
php5:
  Installed: 5.5.9+dfsg-1ubuntu4.21

Here is the core dump:

Reading symbols from /usr/sbin/apache2...(no debugging symbols found)...done.
[New LWP 14926]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/sbin/apache2 -k start'.
Program terminated with signal SIGBUS, Bus error.
#0 0x00007f7d535a97b0 in crc32 () from /lib/x86_64-linux-gnu/libz.so.1
(gdb) bt
#0 0x00007f7d535a97b0 in crc32 () from /lib/x86_64-linux-gnu/libz.so.1
#1 0x00007f7d537c3344 in ?? () from /usr/lib/apache2/modules/mod_deflate.so
#2 0x00007f7d52f9d479 in ?? () from /usr/lib/apache2/modules/mod_filter.so
#3 0x00007f7d52f9d479 in ?? () from /usr/lib/apache2/modules/mod_filter.so
#4 0x00007f7d5654c71f in ?? ()
#5 0x00007f7d5655cf40 in ap_run_handler ()
#6 0x00007f7d5655d489 in ap_invoke_handler ()
#7 0x00007f7d5657251c in ap_internal_redirect ()
#8 0x00007f7d5056bcfc in ?? () from /usr/lib/apache2/modules/mod_rewrite.so
#9 0x00007f7d5655cf40 in ap_run_handler ()
#10 0x00007f7d5655d489 in ap_invoke_handler ()
#11 0x00007f7d56572a5a in ap_process_async_request ()
#12 0x00007f7d56572d34 in ap_process_request ()
#13 0x00007f7d5656f7d2 in ?? ()
#14 0x00007f7d565665b0 in ap_run_process_connection ()
#15 0x00007f7d52b91767 in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#16 0x00007f7d52b919a6 in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#17 0x00007f7d52b9260e in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#18 0x00007f7d5654223e in ap_run_mpm ()
#19 0x00007f7d5653b276 in main ()