I'm using pbzcat against the Wikipedia dump (enwiki-20210101-pages-articles-multistream.xml.bz2) and consuming the output in a Go program. At some point (it's racy, so inconsistent) pbzcat core dumps with a segfault.
If I alter my Go program to consume the output stream without doing any work, I don't see a segfault. My guess is that the segfault happens because pbzcat needs to block and wait for the consumer to catch up. I tried to recreate the issue with smaller data but can't, this happens after a few GBs of decompressed data has been processed.
Here's the Go program doing the work: https://gist.github.com/daaku/1b2c4cfddf6b9ba35586aaaec2d3200f
Here's the segfault output in the logs:
Mar 22 10:02:46 marvin kernel: pbzcat[12166]: segfault at 8 ip 000055a7d84fbbd1 sp 00007f56de0fee00 error 4 in pbzip2[55a7d84f8000+a000]
Mar 22 10:02:46 marvin kernel: Code: 0f 84 f3 01 00 00 89 14 24 ff 15 5a c2 00 00 8b 14 24 85 c0 0f 85 8f 02 00 00 4d 85 ed 0f 85 f6 01 00 00 48 03 2d cf c7 00 00 <8b> 45 08 48 8b 7d 00 48 89 04 24 49 89 fe 49 89 c7 48 85 c0 75 14
Mar 22 10:02:46 marvin systemd[1]: Started Process Core Dump (PID 12184/UID 0).
Mar 22 10:02:47 marvin systemd-coredump[12185]: Process 12151 (pbzcat) of user 1666 dumped core.
Stack trace of thread 12166:
#0 0x000055a7d84fbbd1 n/a (pbzip2 + 0x5bd1)
#1 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#2 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12154:
#0 0x00007f56e5649917 BZ2_bzDecompress (libbz2.so.1.0 + 0xc917)
#1 0x000055a7d84fd746 n/a (pbzip2 + 0x7746)
#2 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#3 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12151:
#0 0x00007f56e562e87c read (libpthread.so.0 + 0x1287c)
#1 0x000055a7d84fad4a n/a (pbzip2 + 0x4d4a)
#2 0x000055a7d8500479 n/a (pbzip2 + 0xa479)
#3 0x000055a7d8500737 n/a (pbzip2 + 0xa737)
#4 0x000055a7d850091b n/a (pbzip2 + 0xa91b)
#5 0x000055a7d84ff41f n/a (pbzip2 + 0x941f)
#6 0x000055a7d84f9d81 n/a (pbzip2 + 0x3d81)
#7 0x00007f56e527fb25 __libc_start_main (libc.so.6 + 0x27b25)
#8 0x000055a7d84fa7fe n/a (pbzip2 + 0x47fe)
Stack trace of thread 12158:
#0 0x00007f56e564654b BZ2_decompress (libbz2.so.1.0 + 0x954b)
#1 0x00007f56e56492cd BZ2_bzDecompress (libbz2.so.1.0 + 0xc2cd)
#2 0x000055a7d84fd746 n/a (pbzip2 + 0x7746)
#3 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#4 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12153:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fb690 n/a (pbzip2 + 0x5690)
#3 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#4 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12160:
#0 0x00007f56e5646636 BZ2_decompress (libbz2.so.1.0 + 0x9636)
#1 0x00007f56e56492cd BZ2_bzDecompress (libbz2.so.1.0 + 0xc2cd)
#2 0x000055a7d84fd746 n/a (pbzip2 + 0x7746)
#3 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#4 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12157:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12162:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12152:
#0 0x00007f56e5295ce2 __sigtimedwait (libc.so.6 + 0x3dce2)
#1 0x00007f56e562f48c sigwait (libpthread.so.0 + 0x1348c)
#2 0x000055a7d84fb5e6 n/a (pbzip2 + 0x55e6)
#3 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#4 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12155:
#0 0x00007f56e5649e8d BZ2_bzDecompress (libbz2.so.1.0 + 0xce8d)
#1 0x000055a7d84fd746 n/a (pbzip2 + 0x7746)
#2 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#3 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12163:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12164:
#0 0x00007f56e5647b24 BZ2_decompress (libbz2.so.1.0 + 0xab24)
#1 0x00007f56e56492cd BZ2_bzDecompress (libbz2.so.1.0 + 0xc2cd)
#2 0x000055a7d84fd746 n/a (pbzip2 + 0x7746)
#3 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#4 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12165:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12161:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12159:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
Stack trace of thread 12156:
#0 0x00007f56e56319ba __futex_abstimed_wait_common64 (libpthread.so.0 + 0x159ba)
#1 0x00007f56e562b260 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0 + 0xf260)
#2 0x000055a7d84fd3ed n/a (pbzip2 + 0x73ed)
#3 0x000055a7d84fd8ee n/a (pbzip2 + 0x78ee)
#4 0x00007f56e5625299 start_thread (libpthread.so.0 + 0x9299)
#5 0x00007f56e5357053 __clone (libc.so.6 + 0xff053)
