util: Fixed crash in HEVC encoding on 32-bit systems

This builds on https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25059, and extends that change to all 32-bit systems.

This fixes a crash on SteamOS with the following test case:

        unsigned char data[] = {
          0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60,
          0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00,
          0x99, 0x2c, 0x0c, 0x01, 0x64, 0x7c, 0x00, 0x7c, 0xd2, 0x56, 0x01, 0x40,
          0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03,
          0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xa0, 0x02,
          0x80, 0x80, 0x32, 0x16, 0x24, 0xbb, 0x90, 0x84, 0x48, 0x9a, 0x83, 0x03,
          0x03, 0x02, 0x00, 0xb2, 0x3e, 0x00, 0x3e, 0x69, 0x2b, 0x00, 0x5f, 0x08,
          0x04, 0x10, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x62, 0x0f, 0x02,
          0x24
        };
        vlVaContext context;
        vlVaBuffer buf;

        memset(&context, 0, sizeof(context));
        memset(&buf, 0, sizeof(buf));
        context.packed_header_emulation_bytes = true;
        buf.data = data;
        buf.size = sizeof(data);
        vlVaHandleVAEncPackedHeaderDataBufferTypeHEVC(&context, &buf);

Cc: mesa-stable
Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31596>
This commit is contained in:
Sam Lantinga
2024-10-11 08:00:52 -07:00
committed by Marge Bot
parent 4ab14de32f
commit 4ed8ef74b4

View File

@@ -346,14 +346,8 @@ vl_vlc_search_byte(struct vl_vlc *vlc, unsigned num_bits, uint8_t value)
static inline void
vl_vlc_removebits(struct vl_vlc *vlc, unsigned pos, unsigned num_bits)
{
#if defined(_MSC_VER)
/* MSVC Compiler defines unsigned long as 4 bytes so use explicit 64 bits mask */
uint64_t lo = (vlc->buffer & (UINT64_MAX >> (pos + num_bits))) << num_bits;
uint64_t hi = (vlc->buffer & (UINT64_MAX << (64 - pos)));
#else
uint64_t lo = (vlc->buffer & (~0UL >> (pos + num_bits))) << num_bits;
uint64_t hi = (vlc->buffer & (~0UL << (64 - pos)));
#endif
vlc->buffer = lo | hi;
vlc->invalid_bits += num_bits;
}