util: Fixed crash in HEVC encoding on 32-bit systems
This builds on https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25059, and extends that change to all 32-bit systems. This fixes a crash on SteamOS with the following test case: unsigned char data[] = { 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0x2c, 0x0c, 0x01, 0x64, 0x7c, 0x00, 0x7c, 0xd2, 0x56, 0x01, 0x40, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xa0, 0x02, 0x80, 0x80, 0x32, 0x16, 0x24, 0xbb, 0x90, 0x84, 0x48, 0x9a, 0x83, 0x03, 0x03, 0x02, 0x00, 0xb2, 0x3e, 0x00, 0x3e, 0x69, 0x2b, 0x00, 0x5f, 0x08, 0x04, 0x10, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x62, 0x0f, 0x02, 0x24 }; vlVaContext context; vlVaBuffer buf; memset(&context, 0, sizeof(context)); memset(&buf, 0, sizeof(buf)); context.packed_header_emulation_bytes = true; buf.data = data; buf.size = sizeof(data); vlVaHandleVAEncPackedHeaderDataBufferTypeHEVC(&context, &buf); Cc: mesa-stable Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31596>
This commit is contained in:
@@ -346,14 +346,8 @@ vl_vlc_search_byte(struct vl_vlc *vlc, unsigned num_bits, uint8_t value)
|
||||
static inline void
|
||||
vl_vlc_removebits(struct vl_vlc *vlc, unsigned pos, unsigned num_bits)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
/* MSVC Compiler defines unsigned long as 4 bytes so use explicit 64 bits mask */
|
||||
uint64_t lo = (vlc->buffer & (UINT64_MAX >> (pos + num_bits))) << num_bits;
|
||||
uint64_t hi = (vlc->buffer & (UINT64_MAX << (64 - pos)));
|
||||
#else
|
||||
uint64_t lo = (vlc->buffer & (~0UL >> (pos + num_bits))) << num_bits;
|
||||
uint64_t hi = (vlc->buffer & (~0UL << (64 - pos)));
|
||||
#endif
|
||||
vlc->buffer = lo | hi;
|
||||
vlc->invalid_bits += num_bits;
|
||||
}
|
||||
|
Reference in New Issue
Block a user