ksvc: wipsvc-vp9-enc

2 files changed, 105 insertions, 45 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_objects.h b/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
index b8302ee8..be933f72 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
@@ -270,6 +270,7 @@ struct _GstVaapiEncPicture
   guint frame_num;
   guint poc;
   guint temporal_id;
+  guint spatial_id;
 #if USE_H264_FEI_ENCODER
   GstVaapiEncFeiMbControl *mbcntrl;
   GstVaapiEncFeiMvPredictor *mvpred;
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_vp9.c b/gst-libs/gst/vaapi/gstvaapiencoder_vp9.c
index c7c0264b..149ee4bd 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_vp9.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_vp9.c
@@ -109,6 +109,8 @@ struct _GstVaapiEncoderVP9
   gboolean k_svc;
   guint num_spatial_layers;
   guint spatial_id;
+  guint spatial_group_count;
+  GstVaapiPictureType previous_frame_type;
   GstVaapiVideoPool *surface_pool_s0;
   GstVaapiVideoPool *surface_pool_s1;
   GstVaapiVideoPool *surface_pool_s2;
@@ -116,7 +118,7 @@ struct _GstVaapiEncoderVP9
   GstVaapiSurfaceProxy *surface_proxy_s0;
   GstVaapiFilter *filter;
 
-  GstVaapiSurfaceProxy *ref_list[GST_VP9_REF_FRAMES];   /* reference list */
+  GstVaapiSurfaceProxy *ref_list[GST_VP9_REF_FRAMES + 1];       /* reference list */
   guint ref_list_idx;           /* next free slot in ref_list */
   GstVaapiEntrypoint entrypoint;
 
@@ -142,10 +144,10 @@ ensure_bitrate (GstVaapiEncoderVP9 * encoder)
     case GST_VAAPI_RATECONTROL_VBR:
     case GST_VAAPI_RATECONTROL_CQP:
 
-      if ((GST_VAAPI_ENCODER_RATE_CONTROL (encoder) == GST_VAAPI_RATECONTROL_CQP) &&
-		      !encoder->software_brc) {
-      	base_encoder->bitrate = 0;
-	break;
+      if ((GST_VAAPI_ENCODER_RATE_CONTROL (encoder) ==
+              GST_VAAPI_RATECONTROL_CQP) && !encoder->software_brc) {
+        base_encoder->bitrate = 0;
+        break;
       }
       if (!base_encoder->bitrate) {
         /* FIXME: Provide better estimation */
@@ -361,11 +363,34 @@ get_ref_indices (guint ref_pic_mode, guint ref_list_idx, guint * last_idx,
 }
 
 static void
-get_ref_indices_k_svc (guint ref_list_idx, guint * last_idx,
-    guint8 * refresh_frame_flags)
+get_ref_indices_k_svc (guint ref_list_idx, guint spatial_group_count,
+    guint * last_idx, guint * gf_idx, guint * arf_idx,
+    guint * ref_frame_ctrl_l0, guint8 * refresh_frame_flags)
 {
-  *last_idx = ref_list_idx - 1;
-  *refresh_frame_flags = 0x01;
+  *ref_frame_ctrl_l0 = 0x07;
+
+  switch (ref_list_idx) {
+    case 0:
+      if (spatial_group_count == 0)
+        *last_idx = *gf_idx = *arf_idx = 0;
+      else
+        *last_idx = *gf_idx = *arf_idx = 1;
+      *refresh_frame_flags = 1 << 1;
+      break;
+    case 1:
+      if (spatial_group_count == 0)
+        *last_idx = *gf_idx = *arf_idx = 1;
+      else
+        *last_idx = *gf_idx = *arf_idx = 2;
+      *refresh_frame_flags = 1 << 2;
+      break;
+    case 2:
+      *last_idx = *gf_idx = *arf_idx = 0;
+      *refresh_frame_flags = 1 << 0;
+      break;
+    default:
+      break;
+  }
 }
 
 static gboolean
@@ -374,9 +399,10 @@ fill_picture_k_svc (GstVaapiEncoderVP9 * encoder,
     GstVaapiCodedBuffer * codedbuf, GstVaapiSurfaceProxy * surface,
     VAEncPictureParameterBufferVP9 * const pic_param)
 {
-  guint last_idx = 0;
+  guint last_idx = 0, gf_idx = 0, arf_idx = 0;
   guint8 refresh_frame_flags = 0;
   guint width, height;
+  guint ref_frame_ctrl_l0 = 0;
 
   width =
       gst_vaapi_surface_get_width (GST_VAAPI_SURFACE_PROXY_SURFACE (surface));
@@ -394,15 +420,14 @@ fill_picture_k_svc (GstVaapiEncoderVP9 * encoder,
   if (picture->type == GST_VAAPI_PICTURE_TYPE_P) {
     pic_param->pic_flags.bits.frame_type = GST_VP9_INTER_FRAME;
 
-    /* only use the last frame for prediction */
-    pic_param->ref_flags.bits.ref_frame_ctrl_l0 = 0x01;
 
-    get_ref_indices_k_svc (encoder->ref_list_idx, &last_idx,
-        &refresh_frame_flags);
+    get_ref_indices_k_svc (encoder->ref_list_idx, encoder->spatial_group_count,
+        &last_idx, &gf_idx, &arf_idx, &ref_frame_ctrl_l0, &refresh_frame_flags);
 
+    pic_param->ref_flags.bits.ref_frame_ctrl_l0 = ref_frame_ctrl_l0;
     pic_param->ref_flags.bits.ref_last_idx = last_idx;
-    pic_param->ref_flags.bits.ref_gf_idx = 0;
-    pic_param->ref_flags.bits.ref_arf_idx = 0;
+    pic_param->ref_flags.bits.ref_gf_idx = gf_idx;
+    pic_param->ref_flags.bits.ref_arf_idx = arf_idx;
     pic_param->refresh_frame_flags = refresh_frame_flags;
   }
 
@@ -460,7 +485,7 @@ fill_picture (GstVaapiEncoderVP9 * encoder,
 
     /* use three of the reference frames (last, golden and altref)
      * for prediction */
-    pic_param->ref_flags.bits.ref_frame_ctrl_l0 = 0x7;
+    pic_param->ref_flags.bits.ref_frame_ctrl_l0 = 0x07;
 
     get_ref_indices (encoder->ref_pic_mode, encoder->ref_list_idx, &last_idx,
         &gf_idx, &arf_idx, &refresh_frame_flags);
@@ -520,6 +545,34 @@ update_ref_list (GstVaapiEncoderVP9 * encoder, GstVaapiEncPicture * picture,
   }
 }
 
+static void
+update_ref_list_k_svc (GstVaapiEncoderVP9 * encoder,
+    GstVaapiEncPicture * picture, GstVaapiSurfaceProxy * ref)
+{
+  guint i;
+  guint index = encoder->spatial_id - 1;
+
+  //update the spatial_group_count
+  if (encoder->spatial_id == 3)
+    ++encoder->spatial_group_count;
+
+  /* update the previous frame type field */
+  encoder->previous_frame_type = picture->type;
+
+  if (picture->type == GST_VAAPI_PICTURE_TYPE_I) {
+    for (i = 0; i < G_N_ELEMENTS (encoder->ref_list); i++)
+      gst_vaapi_surface_proxy_replace (&encoder->ref_list[i], ref);
+    gst_vaapi_surface_proxy_unref (ref);
+    /* set the currently filled slot index */
+    encoder->ref_list_idx = 0;
+    encoder->spatial_group_count = 0;
+    return;
+  }
+  gst_vaapi_surface_proxy_replace (&encoder->ref_list[index], ref);
+  gst_vaapi_surface_proxy_unref (ref);
+  encoder->ref_list_idx = index;
+}
+
 static GstVaapiEncoderStatus
 gst_vaapi_encoder_vp9_encode_k_svc (GstVaapiEncoder * base_encoder,
     GstVaapiEncPicture * picture, GstVaapiCodedBufferProxy * codedbuf)
@@ -530,7 +583,7 @@ gst_vaapi_encoder_vp9_encode_k_svc (GstVaapiEncoder * base_encoder,
   GstVaapiVideoPool *surface_pool;
 
   //k-svc
-  switch (encoder->spatial_id - 1) {
+  switch (picture->spatial_id) {
     case 0:
       reconstruct = gst_vaapi_encoder_create_surface (base_encoder);
       break;
@@ -562,7 +615,7 @@ gst_vaapi_encoder_vp9_encode_k_svc (GstVaapiEncoder * base_encoder,
   if (!gst_vaapi_enc_picture_encode (picture))
     goto error;
 
-  update_ref_list (encoder, picture, reconstruct);
+  update_ref_list_k_svc (encoder, picture, reconstruct);
 
 
   return GST_VAAPI_ENCODER_STATUS_SUCCESS;
@@ -586,8 +639,7 @@ gst_vaapi_encoder_vp9_encode (GstVaapiEncoder * base_encoder,
   GstVaapiSurfaceProxy *reconstruct = NULL;
 
   //Per frame operations for a software brc
-  if (encoder->software_brc)
-  {
+  if (encoder->software_brc) {
     int q = 0, bottom_index = 0, top_index = 0;
 
     //Fixme: Not yet added
@@ -598,9 +650,9 @@ gst_vaapi_encoder_vp9_encode (GstVaapiEncoder * base_encoder,
     if (picture->type == GST_VAAPI_PICTURE_TYPE_P)
       encoder->cpi.common.frame_type = INTER_FRAME;
     vp9_rc_get_one_pass_cbr_params (&encoder->cpi);
-    vp9_set_size_dependent_vars(&encoder->cpi, &q, &bottom_index, &top_index);
+    vp9_set_size_dependent_vars (&encoder->cpi, &q, &bottom_index, &top_index);
     encoder->yac_qi = q;
-    vp9_set_quantizer(&encoder->cpi, q);
+    vp9_set_quantizer (&encoder->cpi, q);
   }
 
   if (!encoder->k_svc || !encoder->num_spatial_layers) {
@@ -625,7 +677,9 @@ gst_vaapi_encoder_vp9_encode (GstVaapiEncoder * base_encoder,
 
   if (encoder->software_brc) {
     gint32 buf_size;
-    buf_size = gst_vaapi_coded_buffer_get_size (GST_VAAPI_CODED_BUFFER_PROXY_BUFFER (codedbuf));
+    buf_size =
+        gst_vaapi_coded_buffer_get_size (GST_VAAPI_CODED_BUFFER_PROXY_BUFFER
+        (codedbuf));
     vp9_rc_postencode_update (&encoder->cpi, buf_size);
     encoder->cpi.common.current_video_frame++;
   }
@@ -705,17 +759,17 @@ gst_vaapi_encoder_vp9_reordering_k_svc (GstVaapiEncoder * base_encoder,
     input_surface_proxy = encoder->surface_proxy_s0;
     input_surface = GST_VAAPI_SURFACE_PROXY_SURFACE (input_surface_proxy);
 
-    g_message ("input_surface: %d %d",
-        gst_vaapi_surface_get_width (input_surface),
-        gst_vaapi_surface_get_height (input_surface));
+    //g_message ("input_surface: %d %d",
+    //    gst_vaapi_surface_get_width (input_surface),
+    //    gst_vaapi_surface_get_height (input_surface));
     output_surface_proxy =
         gst_vaapi_surface_proxy_new_from_pool (GST_VAAPI_SURFACE_POOL
         (surface_pool));
     output_surface = GST_VAAPI_SURFACE_PROXY_SURFACE (output_surface_proxy);
 
-    g_message ("scaled_surface: %d %d",
-        gst_vaapi_surface_get_width (output_surface),
-        gst_vaapi_surface_get_height (output_surface));
+    //g_message ("scaled_surface: %d %d",
+    //    gst_vaapi_surface_get_width (output_surface),
+    //    gst_vaapi_surface_get_height (output_surface));
     filter_status =
         gst_vaapi_filter_process (encoder->filter, input_surface,
         output_surface, 0);
@@ -762,6 +816,7 @@ gst_vaapi_encoder_vp9_reordering_k_svc (GstVaapiEncoder * base_encoder,
   }
 
   encoder->frame_num++;
+  picture->spatial_id = encoder->spatial_id;
   encoder->spatial_id++;
   *output = picture;
   return status;
@@ -818,21 +873,23 @@ software_brc_init_common (GstVaapiEncoder * base_encoder, VP9_COMMON * common)
 }
 
 static void
-software_brc_init_config (GstVaapiEncoder * base_encoder, VP9EncoderConfig *oxcf)
+software_brc_init_config (GstVaapiEncoder * base_encoder,
+    VP9EncoderConfig * oxcf)
 {
   GstVaapiEncoderVP9 *const encoder = GST_VAAPI_ENCODER_VP9 (base_encoder);
-  VP9_COMMON * cm = &(encoder->cpi.common);
+  VP9_COMMON *cm = &(encoder->cpi.common);
 
   oxcf->rc_mode = VPX_CBR;
   oxcf->width = cm->width;
   oxcf->height = cm->height;
   oxcf->bit_depth = VPX_BITS_8;
   oxcf->init_framerate = encoder->frame_rate;
-  if (oxcf->init_framerate > 180) oxcf->init_framerate = 30;
+  if (oxcf->init_framerate > 180)
+    oxcf->init_framerate = 30;
   oxcf->mode = GOOD;
   oxcf->target_bandwidth = base_encoder->bitrate * 1000;
   oxcf->best_allowed_q = vp9_quantizer_to_qindex (0);
-  oxcf->worst_allowed_q = vp9_quantizer_to_qindex(63);
+  oxcf->worst_allowed_q = vp9_quantizer_to_qindex (63);
   oxcf->lag_in_frames = 25;
   oxcf->under_shoot_pct = 25;
   oxcf->over_shoot_pct = 25;
@@ -841,22 +898,22 @@ software_brc_init_config (GstVaapiEncoder * base_encoder, VP9EncoderConfig *oxcf
   oxcf->optimal_buffer_level_ms = 5000;
   oxcf->two_pass_vbrmin_section = 0;
   oxcf->two_pass_vbrmax_section = 2000;
-  oxcf->enable_auto_arf = 0;//Note, double check???
+  oxcf->enable_auto_arf = 0;    //Note, double check???
   oxcf->target_level = 255;
-  if (vp9_get_level_index(oxcf->target_level) >= 0)
-    vp9_config_target_level(oxcf);
+  if (vp9_get_level_index (oxcf->target_level) >= 0)
+    vp9_config_target_level (oxcf);
 
 }
 
 static void
-software_brc_init_comp (GstVaapiEncoder * base_encoder, VP9_COMP *cpi)
+software_brc_init_comp (GstVaapiEncoder * base_encoder, VP9_COMP * cpi)
 {
   VP9EncoderConfig *oxcf = &cpi->oxcf;
-  
+
   cpi->target_level = oxcf->target_level;
 
-  vp9_set_level_constraint(&cpi->level_constraint,
-                       vp9_get_level_index(cpi->target_level));
+  vp9_set_level_constraint (&cpi->level_constraint,
+      vp9_get_level_index (cpi->target_level));
 
   cpi->refresh_golden_frame = 0;
   cpi->refresh_last_frame = 1;
@@ -864,7 +921,8 @@ software_brc_init_comp (GstVaapiEncoder * base_encoder, VP9_COMP *cpi)
 }
 
 static void
-software_brc_init (GstVaapiEncoder *base_encoder){
+software_brc_init (GstVaapiEncoder * base_encoder)
+{
   GstVaapiEncoderVP9 *const encoder = GST_VAAPI_ENCODER_VP9 (base_encoder);
   VP9_COMP *cpi = &encoder->cpi;
   VP9EncoderConfig *oxcf = &cpi->oxcf;
@@ -874,7 +932,7 @@ software_brc_init (GstVaapiEncoder *base_encoder){
   software_brc_init_common (base_encoder, cm);
   software_brc_init_config (base_encoder, oxcf);
   software_brc_init_comp (base_encoder, cpi);
-  
+
   vp9_change_config (cpi, oxcf);
 
   vp9_rc_init_minq_luts ();
@@ -940,8 +998,8 @@ gst_vaapi_encoder_vp9_reconfigure (GstVaapiEncoder * base_encoder)
   if (encoder->software_brc) {
     GstVideoInfo *const vip = GST_VAAPI_ENCODER_VIDEO_INFO (base_encoder);
 
-    encoder->frame_rate =  (double) ((double) GST_VIDEO_INFO_FPS_D (vip) /
-            (double) GST_VIDEO_INFO_FPS_N (vip));
+    encoder->frame_rate = (double) ((double) GST_VIDEO_INFO_FPS_D (vip) /
+        (double) GST_VIDEO_INFO_FPS_N (vip));
 
     gst_vaapi_encoder_set_rate_control (base_encoder,
         GST_VAAPI_RATECONTROL_CQP);
@@ -974,6 +1032,7 @@ gst_vaapi_encoder_vp9_init (GstVaapiEncoderVP9 * encoder)
   encoder->k_svc = FALSE;
   encoder->num_spatial_layers = 3;
   encoder->spatial_id = 0;
+  encoder->spatial_group_count = 0;
 
   memset (encoder->ref_list, 0,
       G_N_ELEMENTS (encoder->ref_list) * sizeof (encoder->ref_list[0]));