vah264enc: Use a FIFO queue to generate DTS

The base parse will infer the DTS by itself, so we need to make DTS
offset before PTS in order to avoid DTS bigger than PTS. We now use
a FIFO queue to store all PTS and assign it to DTS by an offset.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6791>

3 files changed, 83 insertions, 25 deletions

diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
index 74a4473aad..194bdaa0a7 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
@@ -557,6 +557,8 @@ gst_va_base_enc_drain (GstVideoEncoder * venc)
   g_queue_clear_full (&base->ref_list,
       (GDestroyNotify) gst_video_codec_frame_unref);
 
+  gst_queue_array_clear (base->dts_queue);
+
   return GST_FLOW_OK;
 
 error_and_purge_all:
@@ -591,6 +593,8 @@ error_and_purge_all:
   g_queue_clear_full (&base->ref_list,
       (GDestroyNotify) gst_video_codec_frame_unref);
 
+  gst_queue_array_clear (base->dts_queue);
+
   return ret;
 }
 
@@ -744,7 +748,12 @@ gst_va_base_enc_set_format (GstVideoEncoder * venc, GstVideoCodecState * state)
 static gboolean
 gst_va_base_enc_flush (GstVideoEncoder * venc)
 {
+  GstVaBaseEnc *base = GST_VA_BASE_ENC (venc);
+
   _flush_all_frames (GST_VA_BASE_ENC (venc));
+
+  gst_queue_array_clear (base->dts_queue);
+
   return TRUE;
 }
 
@@ -882,14 +891,19 @@ gst_va_base_enc_init (GstVaBaseEnc * self)
   g_queue_init (&self->output_list);
   gst_video_info_init (&self->in_info);
 
+  self->dts_queue = gst_queue_array_new_for_struct (sizeof (GstClockTime), 8);
+
   self->priv = gst_va_base_enc_get_instance_private (self);
 }
 
 static void
 gst_va_base_enc_dispose (GObject * object)
 {
+  GstVaBaseEnc *base = GST_VA_BASE_ENC (object);
+
   _flush_all_frames (GST_VA_BASE_ENC (object));
   gst_va_base_enc_close (GST_VIDEO_ENCODER (object));
+  g_clear_pointer (&base->dts_queue, gst_queue_array_free);
 
   G_OBJECT_CLASS (parent_class)->dispose (object);
 }
@@ -1135,6 +1149,48 @@ gst_va_base_enc_add_codec_tag (GstVaBaseEnc * base, const gchar * codec_name)
 }
 
 void
+gst_va_base_enc_push_dts (GstVaBaseEnc * base,
+    GstVideoCodecFrame * frame, guint max_reorder_num)
+{
+  /* We need to manually insert max_reorder_num slots before the
+     first frame to ensure DTS never bigger than PTS. */
+  if (gst_queue_array_get_length (base->dts_queue) == 0 && max_reorder_num > 0) {
+    GstClockTime dts_diff = 0, dts;
+
+    if (GST_CLOCK_TIME_IS_VALID (frame->duration))
+      dts_diff = frame->duration;
+
+    if (GST_CLOCK_TIME_IS_VALID (base->frame_duration))
+      dts_diff = MAX (base->frame_duration, dts_diff);
+
+    while (max_reorder_num > 0) {
+      if (GST_CLOCK_TIME_IS_VALID (frame->pts)) {
+        dts = frame->pts - dts_diff * max_reorder_num;
+      } else {
+        dts = frame->pts;
+      }
+
+      gst_queue_array_push_tail_struct (base->dts_queue, &dts);
+      max_reorder_num--;
+    }
+  }
+
+  gst_queue_array_push_tail_struct (base->dts_queue, &frame->pts);
+}
+
+GstClockTime
+gst_va_base_enc_pop_dts (GstVaBaseEnc * base)
+{
+  GstClockTime dts;
+
+  g_return_val_if_fail (gst_queue_array_get_length (base->dts_queue) > 0,
+      GST_CLOCK_TIME_NONE);
+
+  dts = *((GstClockTime *) gst_queue_array_pop_head_struct (base->dts_queue));
+  return dts;
+}
+
+void
 gst_va_base_enc_reset_state (GstVaBaseEnc * base)
 {
   GstVaBaseEncClass *klass = GST_VA_BASE_ENC_GET_CLASS (base);
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
index bf5dd52a6c..e0164621b3 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
@@ -23,6 +23,7 @@
 #include "gstvadevice.h"
 #include "gstvaencoder.h"
 #include "gstvaprofile.h"
+#include <gst/base/gstqueuearray.h>
 
 G_BEGIN_DECLS
 
@@ -60,6 +61,7 @@ struct _GstVaBaseEnc
   GQueue reorder_list;
   GQueue ref_list;
   GQueue output_list;
+  GstQueueArray *dts_queue;
 
   GstVideoCodecState *input_state;
   union {
@@ -142,6 +144,10 @@ gint                  gst_va_base_enc_copy_output_data    (GstVaBaseEnc * base,
                                                            GstVaEncodePicture * picture,
                                                            guint8 * data,
                                                            gint size);
+void                  gst_va_base_enc_push_dts            (GstVaBaseEnc * base,
+                                                           GstVideoCodecFrame * frame,
+                                                           guint max_reorder_num);
+GstClockTime          gst_va_base_enc_pop_dts             (GstVaBaseEnc * base);
 void                  gst_va_base_enc_update_property_uint (GstVaBaseEnc * base,
                                                             guint32 * old_val,
                                                             guint32 new_val,
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
index 6f122815f0..a0e5337ace 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
@@ -255,7 +255,6 @@ struct _GstVaH264Enc
     guint cpb_length_bits;
   } rc;
 
-  GstClockTime last_dts;
   GstH264SPS sequence_hdr;
 };
 
@@ -277,7 +276,6 @@ struct _GstVaH264EncFrame
   gint unused_for_reference_pic_num;
 
   gboolean last_frame;
-  gboolean reorder;
 };
 
 /**
@@ -380,7 +378,6 @@ gst_va_enc_frame_new (void)
   frame->unused_for_reference_pic_num = -1;
   frame->picture = NULL;
   frame->last_frame = FALSE;
-  frame->reorder = FALSE;
 
   return frame;
 }
@@ -1552,7 +1549,6 @@ gst_va_h264_enc_reset_state (GstVaBaseEnc * base)
   self->rc.target_bitrate_bits = 0;
   self->rc.cpb_length_bits = 0;
 
-  self->last_dts = GST_CLOCK_TIME_NONE;
   memset (&self->sequence_hdr, 0, sizeof (GstH264SPS));
 }
 
@@ -1883,9 +1879,6 @@ again:
     /* it will unref at pop_frame */
     f = g_queue_pop_nth (&base->reorder_list, index);
     g_assert (f == b_frame);
-
-    if (index > 0)
-      _enc_frame (f)->reorder = TRUE;
   } else {
     b_frame = NULL;
   }
@@ -1914,9 +1907,6 @@ _pop_one_frame (GstVaBaseEnc * base, GstVideoCodecFrame ** out_frame)
   vaframe = _enc_frame (frame);
   if (vaframe->type != GST_H264_B_SLICE) {
     frame = g_queue_pop_tail (&base->reorder_list);
-    if (!g_queue_is_empty (&base->reorder_list))
-      vaframe->reorder = TRUE;
-
     goto get_one;
   }
 
@@ -3079,27 +3069,19 @@ static gboolean
 gst_va_h264_enc_prepare_output (GstVaBaseEnc * base,
     GstVideoCodecFrame * frame, gboolean * complete)
 {
-  GstVaH264Enc *self = GST_VA_H264_ENC (base);
   GstVaH264EncFrame *frame_enc;
   GstBuffer *buf;
 
   frame_enc = _enc_frame (frame);
 
-  if (frame_enc->reorder) {
-    if (!GST_CLOCK_TIME_IS_VALID (self->last_dts)) {
-      GST_WARNING_OBJECT (base, "Reorder frame poc: %d, system frame "
-          "number: %d without previous valid DTS.", frame_enc->poc,
-          frame->system_frame_number);
-      frame->dts = frame->pts;
-    } else {
-      GST_LOG_OBJECT (base, "Set reorder frame poc: %d, system frame "
-          "number: %d with DTS: %" GST_TIME_FORMAT, frame_enc->poc,
-          frame->system_frame_number, GST_TIME_ARGS (self->last_dts));
-      frame->dts = self->last_dts;
-    }
-  } else {
+  frame->dts = gst_va_base_enc_pop_dts (base);
+  if (!GST_CLOCK_TIME_IS_VALID (frame->dts)) {
+    GST_DEBUG_OBJECT (base, "Pop invalid DTS.");
+  } else if (GST_CLOCK_TIME_IS_VALID (frame->pts) && frame->dts > frame->pts) {
+    GST_WARNING_OBJECT (base, "Pop DTS: %" GST_TIME_FORMAT " > PTS: %"
+        GST_TIME_FORMAT, GST_TIME_ARGS (frame->dts),
+        GST_TIME_ARGS (frame->pts));
     frame->dts = frame->pts;
-    self->last_dts = frame->dts;
   }
 
   buf = gst_va_base_enc_create_output_buffer (base,
@@ -3250,14 +3232,27 @@ gst_va_h264_enc_encode_frame (GstVaBaseEnc * base,
 static gboolean
 gst_va_h264_enc_new_frame (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
 {
+  GstVaH264Enc *self = GST_VA_H264_ENC (base);
   GstVaH264EncFrame *frame_in;
 
   frame_in = gst_va_enc_frame_new ();
   gst_video_codec_frame_set_user_data (frame, frame_in, gst_va_enc_frame_free);
 
+  gst_va_base_enc_push_dts (base, frame, self->gop.num_reorder_frames);
+
   return TRUE;
 }
 
+static gboolean
+gst_va_h264_enc_start (GstVideoEncoder * venc)
+{
+  /* Set the minimum pts to some huge value (1000 hours). This keeps
+   * the dts at the start of the stream from needing to be negative. */
+  gst_video_encoder_set_min_pts (venc, GST_SECOND * 60 * 60 * 1000);
+
+  return GST_VIDEO_ENCODER_CLASS (parent_class)->start (venc);
+}
+
 /* *INDENT-OFF* */
 static const gchar *sink_caps_str =
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VA,
@@ -3605,6 +3600,7 @@ gst_va_h264_enc_class_init (gpointer g_klass, gpointer class_data)
   object_class->get_property = gst_va_h264_enc_get_property;
 
   venc_class->flush = GST_DEBUG_FUNCPTR (gst_va_h264_enc_flush);
+  venc_class->start = GST_DEBUG_FUNCPTR (gst_va_h264_enc_start);
 
   va_enc_class->reset_state = GST_DEBUG_FUNCPTR (gst_va_h264_enc_reset_state);
   va_enc_class->reconfig = GST_DEBUG_FUNCPTR (gst_va_h264_enc_reconfig);