diff --git a/src/gallium/frontends/va/picture_hevc_enc.c b/src/gallium/frontends/va/picture_hevc_enc.c
index e266f1835dd..f5f044aca21 100644
--- a/src/gallium/frontends/va/picture_hevc_enc.c
+++ b/src/gallium/frontends/va/picture_hevc_enc.c
@@ -128,6 +128,7 @@ vlVaHandleVAEncSequenceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *con
    VAEncSequenceParameterBufferHEVC *h265 = (VAEncSequenceParameterBufferHEVC *)buf->data;
 
    if (!context->decoder) {
+      context->templat.max_references = 1;
       context->templat.level = h265->general_level_idc;
       context->decoder = drv->pipe->create_video_codec(drv->pipe, &context->templat);