mplayer crashed with SIGSEGV in ff_vdpau_add_data_chunk() while playing 10bit mkv using the ffh264vdpau codec

[Fabien Tassin]

This is in Raring with:
mplayer 2:1.0~rc4.dfsg1+svn34540-1ubuntu2
ffmpeg 6:0.8.5-0ubuntu1
nvidia 310.19-0ubuntu3 (also with 304 and previous).

Playing some-720p-Hi10p.mkv.
libavformat version 53.21.1 (external)
Mismatching header version 53.19.0
libavformat file format detected.
[matroska,webm @ 0x7ffff3d8d8c0]max_analyze_duration reached
[matroska,webm @ 0x7ffff3d8d8c0]Estimating duration from bitrate, this may be inaccurate
[lavf] stream 0: video (h264), -vid 0, h.264
[lavf] stream 1: audio (aac), -aid 0, -alang jpn, aac 2ch
[lavf] stream 2: subtitle (ass), -sid 0, -slang eng, subtitles & notes
[lavf] stream 3: subtitle (ass), -sid 1, -slang eng, subtitles
VIDEO: [H264] 1280x720 0bpp 359.640 fps 0.0 kbps ( 0.0 kbyte/s)
Load subtitles in ./
==========================================================================
Forced video codec: ffmpeg12vdpau
Forced video codec: ffh264vdpau
Opening video decoder: [ffmpeg] FFmpeg's libavcodec codec family
libavcodec version 53.35.0 (external)
Mismatching header version 53.32.2
Selected video codec: [ffh264vdpau] vfm: ffmpeg (FFmpeg H.264 (VDPAU))
==========================================================================
==========================================================================
Opening audio decoder: [ffmpeg] FFmpeg/libavcodec audio decoders
AUDIO: 48000 Hz, 2 ch, s16le, 0.0 kbit/0.00% (ratio: 0->192000)
Selected audio codec: [ffaac] afm: ffmpeg (FFmpeg AAC (MPEG-2/MPEG-4 Audio))
==========================================================================
[New Thread 0x7fffdec8c700 (LWP 2851)]
AO: [pulse] 48000Hz 2ch s16le (2 bytes per sample)
Starting playback...

Program received signal SIGSEGV, Segmentation fault.
ff_vdpau_add_data_chunk (s=s@entry=0xe2ae80, buf=buf@entry=0x7ffff31fdd7a <start_code.10388> "", buf_size=buf_size@entry=3)
    at /build/buildd/libav-0.8.5/libavcodec/vdpau.c:125
125 /build/buildd/libav-0.8.5/libavcodec/vdpau.c: No such file or directory.
(gdb) bt
#0 ff_vdpau_add_data_chunk (s=s@entry=0xe2ae80, buf=buf@entry=0x7ffff31fdd7a <start_code.10388> "", buf_size=buf_size@entry=3)
    at /build/buildd/libav-0.8.5/libavcodec/vdpau.c:125
#1 0x00007ffff2ead0a8 in decode_nal_units (h=h@entry=0xe2ae80, buf=0x232e340 "", buf_size=buf_size@entry=48949)
    at /build/buildd/libav-0.8.5/libavcodec/h264.c:3995
#2 0x00007ffff2ead129 in decode_frame (avctx=0xcbb9e0, data=0xde5840, data_size=0x7fffffffd224, avpkt=<optimized out>)
    at /build/buildd/libav-0.8.5/libavcodec/h264.c:4178
#3 0x00007ffff30a51f0 in avcodec_decode_video2 (avctx=avctx@entry=0xcbb9e0, picture=picture@entry=0xde5840,
    got_picture_ptr=got_picture_ptr@entry=0x7fffffffd224, avpkt=avpkt@entry=0x7fffffffd230) at /build/buildd/libav-0.8.5/libavcodec/utils.c:1152
#4 0x0000000000594c1a in decode (sh=0xc84570, data=<optimized out>, len=48949, flags=<optimized out>) at libmpcodecs/vd_ffmpeg.c:808
#5 0x00000000004e3ef0 in decode_video (sh_video=sh_video@entry=0xc84570, start=0x232e340 "", in_size=48949, drop_frame=drop_frame@entry=0,
    pts=<optimized out>, full_frame=full_frame@entry=0x0) at libmpcodecs/dec_video.c:393
#6 0x000000000045d9e8 in generate_video_frame (d_video=0xad0f90, sh_video=0xc84570) at mplayer.c:1811
#7 update_video (blit_frame=blit_frame@entry=0x7fffffffd424) at mplayer.c:2465
#8 0x0000000000450e67 in main (argc=<optimized out>, argv=<optimized out>) at mplayer.c:3729
(gdb) bt f
#0 ff_vdpau_add_data_chunk (s=s@entry=0xe2ae80, buf=buf@entry=0x7ffff31fdd7a <start_code.10388> "", buf_size=buf_size@entry=3)
    at /build/buildd/libav-0.8.5/libavcodec/vdpau.c:125
        render = 0xfb92a0
#1 0x00007ffff2ead0a8 in decode_nal_units (h=h@entry=0xe2ae80, buf=0x232e340 "", buf_size=buf_size@entry=48949)
    at /build/buildd/libav-0.8.5/libavcodec/h264.c:3995
        start_code = "\000\000\001"
        consumed = 48254
        dst_length = 48253
        bit_length = 386023
        i = <optimized out>
        err = <optimized out>
        ptr = 0x232e5f8 "\210\202"
        nalsize = 48254
        s = 0xe2ae80
        avctx = 0xcbb9e0
        hx = 0xe2ae80
        buf_index = 48949
        context_count = 0
        next_avc = 48949
        pass = 1
        nals_needed = 0
        nal_index = 2
#2 0x00007ffff2ead129 in decode_frame (avctx=0xcbb9e0, data=0xde5840, data_size=0x7fffffffd224, avpkt=<optimized out>)
    at /build/buildd/libav-0.8.5/libavcodec/h264.c:4178
        buf = <optimized out>
        buf_size = 48949
        h = 0xe2ae80
        s = 0xe2ae80
        pict = 0xde5840
        buf_index = 0
#3 0x00007ffff30a51f0 in avcodec_decode_video2 (avctx=avctx@entry=0xcbb9e0, picture=picture@entry=0xde5840,
    got_picture_ptr=got_picture_ptr@entry=0x7fffffffd224, avpkt=avpkt@entry=0x7fffffffd230) at /build/buildd/libav-0.8.5/libavcodec/utils.c:1152
        ret = <optimized out>
#4 0x0000000000594c1a in decode (sh=0xc84570, data=<optimized out>, len=48949, flags=<optimized out>) at libmpcodecs/vd_ffmpeg.c:808
        got_picture = 0
        ret = <optimized out>
        ctx = 0xe256c0
        pic = 0xde5840
        avctx = 0xcbb9e0
        mpi = <optimized out>
        dr1 = <optimized out>
        pkt = {pts = -9223372036854775808, dts = -9223372036854775808, data = 0x232e340 "", size = 48949, stream_index = 0, flags = 1,
          side_data = 0xd17630, side_data_elems = 1, duration = 0, destruct = 0x0, priv = 0x7fffe72dc2a0, pos = -1, convergence_duration = 0}
#5 0x00000000004e3ef0 in decode_video (sh_video=sh_video@entry=0xc84570, start=0x232e340 "", in_size=48949, drop_frame=drop_frame@entry=0,
    pts=<optimized out>, full_frame=full_frame@entry=0x0) at libmpcodecs/dec_video.c:393
        mpi = 0x0
        t = 4071840270
        t2 = <optimized out>
        tt = <optimized out>
        delay = 0
        got_picture = 1
#6 0x000000000045d9e8 in generate_video_frame (d_video=0xad0f90, sh_video=0xc84570) at mplayer.c:1811
        drop_frame = 0
        decoded_frame = <optimized out>
        start = 0x232e340 ""
        in_size = <optimized out>
        hit_eof = 0
        pts = 0
#7 update_video (blit_frame=blit_frame@entry=0x7fffffffd424) at mplayer.c:2465
        sh_video = 0xc84570
        frame_time = <optimized out>
#8 0x0000000000450e67 in main (argc=<optimized out>, argv=<optimized out>) at mplayer.c:3729
        frame_time = <optimized out>
        frame_time_remaining = 0
        blit_frame = 0
        skip_timing = 1
        aq_sleep_time = 0
        opt_exit = <optimized out>
        profile_config_loaded = <optimized out>
        i = <optimized out>
(gdb)

it doesn't crash with regular (non-10-bit) mkvs though.

ProblemType: Bug
DistroRelease: Ubuntu 13.04
Package: mplayer 2:1.0~rc4.dfsg1+svn34540-1ubuntu2
ProcVersionSignature: Ubuntu 3.8.0-2.6-generic 3.8.0-rc4
Uname: Linux 3.8.0-2-generic x86_64
NonfreeKernelModules: nvidia
ApportVersion: 2.8-0ubuntu3
Architecture: amd64
Date: Thu Jan 31 20:53:00 2013
InstallationDate: Installed on 2012-12-16 (46 days ago)
InstallationMedia: Ubuntu 12.04.1 LTS "Precise Pangolin" - Release amd64 (20120823.1)
MarkForUpload: True
SourcePackage: mplayer
UpgradeStatus: No upgrade log present (probably fresh install)