From 4ed8ef74b4dc111425d6596eb3341d91d563bf00 Mon Sep 17 00:00:00 2001 From: Sam Lantinga Date: Fri, 11 Oct 2024 08:00:52 -0700 Subject: [PATCH] util: Fixed crash in HEVC encoding on 32-bit systems This builds on https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25059, and extends that change to all 32-bit systems. This fixes a crash on SteamOS with the following test case: unsigned char data[] = { 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0x2c, 0x0c, 0x01, 0x64, 0x7c, 0x00, 0x7c, 0xd2, 0x56, 0x01, 0x40, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xa0, 0x02, 0x80, 0x80, 0x32, 0x16, 0x24, 0xbb, 0x90, 0x84, 0x48, 0x9a, 0x83, 0x03, 0x03, 0x02, 0x00, 0xb2, 0x3e, 0x00, 0x3e, 0x69, 0x2b, 0x00, 0x5f, 0x08, 0x04, 0x10, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x62, 0x0f, 0x02, 0x24 }; vlVaContext context; vlVaBuffer buf; memset(&context, 0, sizeof(context)); memset(&buf, 0, sizeof(buf)); context.packed_header_emulation_bytes = true; buf.data = data; buf.size = sizeof(data); vlVaHandleVAEncPackedHeaderDataBufferTypeHEVC(&context, &buf); Cc: mesa-stable Reviewed-by: Bas Nieuwenhuizen Part-of: --- src/util/vl_vlc.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/util/vl_vlc.h b/src/util/vl_vlc.h index d3a2a54deb6..a8a001bcfe1 100644 --- a/src/util/vl_vlc.h +++ b/src/util/vl_vlc.h @@ -346,14 +346,8 @@ vl_vlc_search_byte(struct vl_vlc *vlc, unsigned num_bits, uint8_t value) static inline void vl_vlc_removebits(struct vl_vlc *vlc, unsigned pos, unsigned num_bits) { -#if defined(_MSC_VER) - /* MSVC Compiler defines unsigned long as 4 bytes so use explicit 64 bits mask */ uint64_t lo = (vlc->buffer & (UINT64_MAX >> (pos + num_bits))) << num_bits; uint64_t hi = (vlc->buffer & (UINT64_MAX << (64 - pos))); -#else - uint64_t lo = (vlc->buffer & (~0UL >> (pos + num_bits))) << num_bits; - uint64_t hi = (vlc->buffer & (~0UL << (64 - pos))); -#endif vlc->buffer = lo | hi; vlc->invalid_bits += num_bits; }