diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_dec.c b/src/gallium/drivers/radeonsi/radeon_vcn_dec.c
index 8974db14eed..b7b5ffda5e9 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_dec.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_dec.c
@@ -2740,8 +2740,10 @@ static void radeon_dec_destroy(struct pipe_video_codec *decoder)
       send_msg_buf(dec);
       flush(dec, 0, &dec->destroy_fence);
       dec->ws->fence_wait(dec->ws, dec->destroy_fence, PIPE_DEFAULT_DECODER_FEEDBACK_TIMEOUT_NS);
+      dec->ws->fence_reference(&dec->destroy_fence, NULL);
    }
 
+   dec->ws->fence_reference(&dec->prev_fence, NULL);
    dec->ws->cs_destroy(&dec->cs);
 
    if (dec->stream_type == RDECODE_CODEC_JPEG) {
@@ -2919,9 +2921,9 @@ static void radeon_dec_end_frame(struct pipe_video_codec *decoder, struct pipe_v
       return;
 
    dec->send_cmd(dec, target, picture);
-   flush(dec, PIPE_FLUSH_ASYNC, picture->fence);
+   flush(dec, PIPE_FLUSH_ASYNC, &dec->prev_fence);
    if (picture->fence)
-      dec->prev_fence = *picture->fence;
+      *picture->fence = dec->prev_fence;
    next_buffer(dec);
 }
 
@@ -2965,6 +2967,9 @@ static int radeon_dec_get_decoder_fence(struct pipe_video_codec *decoder,
                                         uint64_t timeout) {
 
    struct radeon_decoder *dec = (struct radeon_decoder *)decoder;
+   /* Only last fence is valid */
+   if (fence != dec->prev_fence)
+      return true;
    return dec->ws->fence_wait(dec->ws, fence, timeout);
 }