videodecoder/videoencoder: Mark pads as requiring reconfiguration again if negotiation fails

Otherwise we might end up in non-optimal configuration, especially
when a flush happened during reconfiguration.

2 files changed, 16 insertions, 4 deletions

diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 281c420eb..1f33f0e6c 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -2435,6 +2435,7 @@ gst_video_decoder_finish_frame (GstVideoDecoder * decoder,
   if (G_UNLIKELY (priv->output_state_changed || (priv->output_state
               && gst_pad_check_reconfigure (decoder->srcpad)))) {
     if (!gst_video_decoder_negotiate (decoder)) {
+      gst_pad_mark_reconfigure (decoder->srcpad);
       if (GST_PAD_IS_FLUSHING (decoder->srcpad))
         ret = GST_FLOW_FLUSHING;
       else
@@ -3182,6 +3183,7 @@ gst_video_decoder_allocate_output_buffer (GstVideoDecoder * decoder)
           || gst_pad_check_reconfigure (decoder->srcpad))) {
     if (!gst_video_decoder_negotiate (decoder)) {
       GST_DEBUG_OBJECT (decoder, "Failed to negotiate, fallback allocation");
+      gst_pad_mark_reconfigure (decoder->srcpad);
       goto fallback;
     }
   }
@@ -3246,8 +3248,12 @@ gst_video_decoder_allocate_output_frame (GstVideoDecoder *
   }
 
   if (G_UNLIKELY (decoder->priv->output_state_changed
-          || gst_pad_check_reconfigure (decoder->srcpad)))
-    gst_video_decoder_negotiate (decoder);
+          || gst_pad_check_reconfigure (decoder->srcpad))) {
+    if (!gst_video_decoder_negotiate (decoder)) {
+      GST_DEBUG_OBJECT (decoder, "Failed to negotiate, fallback allocation");
+      gst_pad_mark_reconfigure (decoder->srcpad);
+    }
+  }
 
   GST_LOG_OBJECT (decoder, "alloc buffer size %d", num_bytes);
 
diff --git a/gst-libs/gst/video/gstvideoencoder.c b/gst-libs/gst/video/gstvideoencoder.c
index 3c06ea997..f5c183202 100644
--- a/gst-libs/gst/video/gstvideoencoder.c
+++ b/gst-libs/gst/video/gstvideoencoder.c
@@ -1619,6 +1619,7 @@ gst_video_encoder_allocate_output_buffer (GstVideoEncoder * encoder, gsize size)
               && gst_pad_check_reconfigure (encoder->srcpad)))) {
     if (!gst_video_encoder_negotiate (encoder)) {
       GST_DEBUG_OBJECT (encoder, "Failed to negotiate, fallback allocation");
+      gst_pad_mark_reconfigure (encoder->srcpad);
       goto fallback;
     }
   }
@@ -1667,8 +1668,12 @@ gst_video_encoder_allocate_output_frame (GstVideoEncoder *
   GST_VIDEO_ENCODER_STREAM_LOCK (encoder);
   if (G_UNLIKELY (encoder->priv->output_state_changed
           || (encoder->priv->output_state
-              && gst_pad_check_reconfigure (encoder->srcpad))))
-    gst_video_encoder_negotiate (encoder);
+              && gst_pad_check_reconfigure (encoder->srcpad)))) {
+    if (!gst_video_encoder_negotiate (encoder)) {
+      GST_DEBUG_OBJECT (encoder, "Failed to negotiate, fallback allocation");
+      gst_pad_mark_reconfigure (encoder->srcpad);
+    }
+  }
 
   GST_LOG_OBJECT (encoder, "alloc buffer size %" G_GSIZE_FORMAT, size);
 
@@ -1740,6 +1745,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
   if (G_UNLIKELY (priv->output_state_changed || (priv->output_state
               && gst_pad_check_reconfigure (encoder->srcpad)))) {
     if (!gst_video_encoder_negotiate (encoder)) {
+      gst_pad_mark_reconfigure (encoder->srcpad);
       if (GST_PAD_IS_FLUSHING (encoder->srcpad))
         ret = GST_FLOW_FLUSHING;
       else