videoparsers: h264: Interpolate PTS based on POCh264-pts

https://bugzilla.gnome.org/show_bug.cgi?id=659489

2 files changed, 120 insertions, 27 deletions

diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index b160eb9da..8410a313d 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -492,6 +492,7 @@ gst_h264_parse_reset (GstH264Parse * h264parse)
   h264parse->have_sps = FALSE;
 
   h264parse->dts = GST_CLOCK_TIME_NONE;
+  h264parse->pts = GST_CLOCK_TIME_NONE;
   h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
   h264parse->do_ts = TRUE;
 
@@ -501,7 +502,8 @@ gst_h264_parse_reset (GstH264Parse * h264parse)
   gst_event_replace (&h264parse->force_key_unit_event, NULL);
 
   h264parse->discont = FALSE;
-
+  h264parse->frame_count = -1;
+  h264parse->poc_base = 0;
   gst_h264_parse_reset_frame (h264parse);
 }
 
@@ -516,6 +518,7 @@ gst_h264_parse_start (GstBaseParse * parse)
   h264parse->nalparser = gst_h264_nal_parser_new ();
 
   h264parse->dts = GST_CLOCK_TIME_NONE;
+  h264parse->pts = GST_CLOCK_TIME_NONE;
   h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
   h264parse->sei_pic_struct_pres_flag = FALSE;
   h264parse->sei_pic_struct = 0;
@@ -776,6 +779,8 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
             sei.payload.pic_timing.pic_struct_present_flag;
         h264parse->sei_cpb_removal_delay =
             sei.payload.pic_timing.cpb_removal_delay;
+        h264parse->sei_dpb_output_delay =
+            sei.payload.pic_timing.dpb_output_delay;
         if (h264parse->sei_pic_struct_pres_flag)
           h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
         GST_LOG_OBJECT (h264parse, "pic timing updated");
@@ -941,9 +946,6 @@ gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
 
           h264parse->is_ref_pic = nalu->ref_idc;
 
-          if (nal_type == GST_H264_NAL_SLICE_IDR)
-            h264parse->idrframe = TRUE;
-
           h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
           h264parse->field_pic_flag = slice.field_pic_flag;
 
@@ -952,6 +954,11 @@ gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
                 h264parse->pic_order_cnt.frame_num;
             h264parse->pic_order_cnt.frame_num = slice.frame_num;
 
+            if (nal_type == GST_H264_NAL_SLICE_IDR)
+              h264parse->idrframe = TRUE;
+
+            h264parse->frame_count++;
+
             /* Initialize picture structure */
             if (!slice.field_pic_flag)
               h264parse->pic_structure = GST_H264_PICTURE_STRUCTURE_FRAME;
@@ -965,7 +972,6 @@ gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
 
             init_picture_poc (h264parse, &slice);
             update_picture_poc (h264parse, &slice);
-
             GST_DEBUG_OBJECT (h264parse, "POC = %d", h264parse->POC);
           }
         }
@@ -1682,25 +1688,33 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
 
 static void
 gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
-    GstClockTime * out_ts, GstClockTime * out_dur, gboolean frame)
+    GstClockTime * out_dts, GstClockTime * out_pts, GstClockTime * out_dur,
+    gboolean frame)
 {
   GstH264SPS *sps = h264parse->nalparser->last_sps;
-  GstClockTime upstream;
-  gint duration = 1;
+  GstClockTime upstream_dts, upstream_pts;
+  GstClockTime out_display_dur = GST_CLOCK_TIME_NONE;
+  gint duration = 1, display_duration = -1;
+  gboolean interpolate_ts;
 
   g_return_if_fail (out_dur != NULL);
-  g_return_if_fail (out_ts != NULL);
+  g_return_if_fail (out_dts != NULL);
+  g_return_if_fail (out_pts != NULL);
 
-  upstream = *out_ts;
+  upstream_dts = *out_dts;
+  upstream_pts = *out_pts;
 
   if (!frame) {
     GST_LOG_OBJECT (h264parse, "no frame data ->  0 duration");
     *out_dur = 0;
     goto exit;
   } else {
-    *out_ts = upstream;
+    *out_dts = upstream_dts;
+    *out_pts = upstream_pts;
   }
 
+  duration = h264parse->field_pic_flag ? 1 : 2;
+
   if (!sps) {
     GST_DEBUG_OBJECT (h264parse, "referred SPS invalid");
     goto exit;
@@ -1719,6 +1733,8 @@ gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
     goto exit;
   }
 
+  interpolate_ts = TRUE;
+
   if (h264parse->sei_pic_struct_pres_flag &&
       h264parse->sei_pic_struct != (guint8) - 1) {
     /* Note that when h264parse->sei_pic_struct == -1 (unspecified), there
@@ -1730,22 +1746,22 @@ gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
     switch (h264parse->sei_pic_struct) {
       case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
       case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
-        duration = 1;
+        display_duration = 1;
         break;
       case GST_H264_SEI_PIC_STRUCT_FRAME:
       case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
       case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
-        duration = 2;
+        display_duration = 2;
         break;
       case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
       case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
-        duration = 3;
+        display_duration = 3;
         break;
       case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
-        duration = 4;
+        display_duration = 4;
         break;
       case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
-        duration = 6;
+        display_duration = 6;
         break;
       default:
         GST_DEBUG_OBJECT (h264parse,
@@ -1754,10 +1770,11 @@ gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
         break;
     }
   } else {
-    duration = h264parse->field_pic_flag ? 1 : 2;
+    display_duration = duration;
   }
 
   GST_LOG_OBJECT (h264parse, "frame tick duration %d", duration);
+  GST_LOG_OBJECT (h264parse, "frame display duration %d", display_duration);
 
   /*
    * h264parse.264 C.1.2 Timing of coded picture removal (equivalent to DTS):
@@ -1768,24 +1785,49 @@ gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
    */
 
   if (h264parse->ts_trn_nb != GST_CLOCK_TIME_NONE) {
+    GstClockTime display_dur;
+
     GST_LOG_OBJECT (h264parse, "buffering based ts");
     /* buffering period is present */
-    if (upstream != GST_CLOCK_TIME_NONE) {
-      /* If upstream timestamp is valid, we respect it and adjust current
+    if (upstream_dts != GST_CLOCK_TIME_NONE) {
+      /* If upstream dts is valid, we respect it and adjust current
        * reference point */
-      h264parse->ts_trn_nb = upstream -
+      h264parse->ts_trn_nb = upstream_dts -
           (GstClockTime) gst_util_uint64_scale_int
           (h264parse->sei_cpb_removal_delay * GST_SECOND,
           sps->vui_parameters.num_units_in_tick,
           sps->vui_parameters.time_scale);
     } else {
-      /* If no upstream timestamp is given, we write in new timestamp */
-      upstream = h264parse->dts = h264parse->ts_trn_nb +
+      /* If no upstream dts is given, we write in new dts */
+      upstream_dts = h264parse->dts = h264parse->ts_trn_nb +
           (GstClockTime) gst_util_uint64_scale_int
           (h264parse->sei_cpb_removal_delay * GST_SECOND,
           sps->vui_parameters.num_units_in_tick,
           sps->vui_parameters.time_scale);
     }
+
+    if (upstream_pts != GST_CLOCK_TIME_NONE) {
+      h264parse->pts = upstream_pts;
+    } else {
+      /* If no upstream pts is given, we write new pts */
+      upstream_pts = h264parse->pts = h264parse->dts +
+          (GstClockTime) gst_util_uint64_scale_int
+          (h264parse->sei_dpb_output_delay * GST_SECOND,
+          sps->vui_parameters.num_units_in_tick,
+          sps->vui_parameters.time_scale);
+    }
+
+    /* frame display duration */
+    display_dur = gst_util_uint64_scale_int (display_duration * GST_SECOND,
+        sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
+    /* sanity check for frame display duration */
+    if (display_dur < GST_MSECOND) {
+      GST_DEBUG_OBJECT (h264parse, "discarding display_dur %" GST_TIME_FORMAT,
+          GST_TIME_ARGS (display_dur));
+    } else {
+      out_display_dur = display_dur;
+    }
+
   } else {
     GstClockTime dur;
 
@@ -1794,22 +1836,67 @@ gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
      * track upstream timestamp and provide best guess frame duration */
     dur = gst_util_uint64_scale_int (duration * GST_SECOND,
         sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
-    /* sanity check */
+
+    /* sanity check for frame duration */
     if (dur < GST_MSECOND) {
       GST_DEBUG_OBJECT (h264parse, "discarding dur %" GST_TIME_FORMAT,
           GST_TIME_ARGS (dur));
     } else {
       *out_dur = dur;
     }
+
+    if (upstream_pts == GST_CLOCK_TIME_NONE) {
+      guint32 poc = h264parse->POC;
+
+      if (display_duration >= 3) {
+        GST_WARNING ("Not able to interpolate the PTS, if pic_struct >= 3");
+        goto exit;
+      }
+
+      if (h264parse->idrframe)
+        h264parse->poc_base = h264parse->frame_count;
+
+      if (!poc)
+        poc = h264parse->poc_base;
+      else if (!h264parse->field_pic_flag)
+        poc = h264parse->poc_base + (poc / 2);
+      else
+        poc = h264parse->poc_base + poc;
+
+      if (!GST_CLOCK_TIME_IS_VALID (h264parse->pts)) {
+        upstream_pts = 0;
+      } else {
+        /* poc based pts calculation */
+        upstream_pts =
+            (GstClockTime) gst_util_uint64_scale_int
+            (poc * duration * GST_SECOND,
+            sps->vui_parameters.num_units_in_tick,
+            sps->vui_parameters.time_scale);
+      }
+    }
   }
 
 exit:
-  if (GST_CLOCK_TIME_IS_VALID (upstream))
-    *out_ts = h264parse->dts = upstream;
+  /******** Set DTS ********/
+  if (GST_CLOCK_TIME_IS_VALID (upstream_dts))
+    *out_dts = h264parse->dts = upstream_dts;
+  else if (GST_CLOCK_TIME_IS_VALID (h264parse->dts))
+    *out_dts = h264parse->dts;
+  else if (interpolate_ts)
+    *out_dts = h264parse->dts = 0;
 
   if (GST_CLOCK_TIME_IS_VALID (*out_dur) &&
       GST_CLOCK_TIME_IS_VALID (h264parse->dts))
     h264parse->dts += *out_dur;
+
+
+  /******** Set PTS ********/
+  if (GST_CLOCK_TIME_IS_VALID (upstream_pts))
+    *out_pts = h264parse->pts = upstream_pts;
+
+  /******** Set Duration ********/
+  if (GST_CLOCK_TIME_IS_VALID (out_display_dur))
+    *out_dur = out_display_dur;
 }
 
 static GstFlowReturn
@@ -1828,8 +1915,8 @@ gst_h264_parse_parse_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
    * particularly since our ts not that good they handle seeking etc */
   if (h264parse->do_ts)
     gst_h264_parse_get_timestamp (h264parse,
-        &GST_BUFFER_TIMESTAMP (buffer), &GST_BUFFER_DURATION (buffer),
-        h264parse->frame_start);
+        &GST_BUFFER_DTS (buffer), &GST_BUFFER_PTS (buffer),
+        &GST_BUFFER_DURATION (buffer), h264parse->frame_start);
 
   if (h264parse->keyframe)
     GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
@@ -2462,6 +2549,7 @@ gst_h264_parse_event (GstBaseParse * parse, GstEvent * event)
     }
     case GST_EVENT_FLUSH_STOP:
       h264parse->dts = GST_CLOCK_TIME_NONE;
+      h264parse->pts = GST_CLOCK_TIME_NONE;
       h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
 
       res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
diff --git a/gst/videoparsers/gsth264parse.h b/gst/videoparsers/gsth264parse.h
index 27da7be88..69d841d01 100644
--- a/gst/videoparsers/gsth264parse.h
+++ b/gst/videoparsers/gsth264parse.h
@@ -109,6 +109,7 @@ struct _GstH264Parse
 
   /* Infos we need to keep track of */
   guint32 sei_cpb_removal_delay;
+  guint32 sei_dpb_output_delay;
   guint8 sei_pic_struct;
   guint8 sei_pic_struct_pres_flag;
   guint field_pic_flag;
@@ -116,6 +117,8 @@ struct _GstH264Parse
   /* cached timestamps */
   /* (trying to) track upstream dts and interpolate */
   GstClockTime dts;
+  /* (trying to) track upstream pts and interpolate */
+  GstClockTime pts;
   /* dts at start of last buffering period */
   GstClockTime ts_trn_nb;
   gboolean do_ts;
@@ -128,6 +131,8 @@ struct _GstH264Parse
   guint pic_structure;
   gboolean is_ref_pic;
   gboolean idrframe;
+  gint poc_base;
+  gint frame_count;
 
   /* frame parsing */
   /*guint last_nal_pos;*/