libs: encoder: h265: choose the profile based on allowed list.

We can decide the profile in ensure_profile(), based on allowed list
passed by the encode. We also need to check whether the entrypoint is
available. Once it is decided, no need to check the hw entrypoint
them again.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/349>

1 files changed, 64 insertions, 79 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
index 4cc274cd..49745e46 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
@@ -1057,95 +1057,84 @@ _check_vps_sps_pps_status (GstVaapiEncoderH265 * encoder,
   }
 }
 
-/* Derives the profile supported by the underlying hardware */
 static gboolean
-ensure_hw_profile (GstVaapiEncoderH265 * encoder)
+is_profile_allowed (GstVaapiEncoderH265 * encoder, GstVaapiProfile profile)
 {
-  GstVaapiDisplay *const display = GST_VAAPI_ENCODER_DISPLAY (encoder);
-  GstVaapiEntrypoint entrypoint = encoder->entrypoint;
-  GstVaapiProfile profile, profiles[4];
-  guint i, num_profiles = 0;
-
-  profiles[num_profiles++] = encoder->profile;
-  switch (encoder->profile) {
-    case GST_VAAPI_PROFILE_H265_MAIN_STILL_PICTURE:
-      profiles[num_profiles++] = GST_VAAPI_PROFILE_H265_MAIN;
-      // fall-through
-    case GST_VAAPI_PROFILE_H265_MAIN:
-      profiles[num_profiles++] = GST_VAAPI_PROFILE_H265_MAIN10;
-      break;
-    default:
-      break;
-  }
-
-  profile = GST_VAAPI_PROFILE_UNKNOWN;
-  for (i = 0; i < num_profiles; i++) {
-    if (gst_vaapi_display_has_encoder (display, profiles[i], entrypoint)) {
-      profile = profiles[i];
-      break;
-    }
-  }
-  if (profile == GST_VAAPI_PROFILE_UNKNOWN)
-    goto error_unsupported_profile;
-
-  GST_VAAPI_ENCODER_CAST (encoder)->profile = profile;
-  return TRUE;
-
-  /* ERRORS */
-error_unsupported_profile:
-  {
-    GST_ERROR ("unsupported HW profile %s",
-        gst_vaapi_profile_get_va_name (encoder->profile));
-    return FALSE;
-  }
-}
-
-/* Check target decoder constraints */
-static gboolean
-ensure_profile_limits (GstVaapiEncoderH265 * encoder)
-{
-  gint i;
+  guint i;
 
-  if (!encoder->allowed_profiles)
+  if (encoder->allowed_profiles == NULL)
     return TRUE;
 
-  for (i = 0; i < encoder->allowed_profiles->len; i++) {
-    if (encoder->profile ==
+  for (i = 0; i < encoder->allowed_profiles->len; i++)
+    if (profile ==
         g_array_index (encoder->allowed_profiles, GstVaapiProfile, i))
       return TRUE;
-  }
 
-  GST_WARNING
-      ("Needs to lower coding tools to meet target decoder constraints");
-  GST_WARNING ("Only supporting Main profile, reset profile to Main");
-
-  encoder->profile = GST_VAAPI_PROFILE_H265_MAIN;
-  encoder->profile_idc =
-      gst_vaapi_utils_h265_get_profile_idc (encoder->profile);
-
-  return TRUE;
+  return FALSE;
 }
 
-/* Derives the minimum profile from the active coding tools */
+/* Derives the profile from the active coding tools. */
 static gboolean
 ensure_profile (GstVaapiEncoderH265 * encoder)
 {
   GstVaapiProfile profile;
   const GstVideoFormat format =
       GST_VIDEO_INFO_FORMAT (GST_VAAPI_ENCODER_VIDEO_INFO (encoder));
+  guint depth, chrome;
+  GstVaapiProfile profile_candidates[3];
+  guint num, i;
+
+  g_assert (GST_VIDEO_FORMAT_INFO_IS_YUV (gst_video_format_get_info (format)));
+  depth = GST_VIDEO_FORMAT_INFO_DEPTH (gst_video_format_get_info (format), 0);
+  chrome = gst_vaapi_utils_h265_get_chroma_format_idc
+      (gst_vaapi_video_format_get_chroma_type (format));
+
+  num = 0;
+
+  if (chrome == 3) {
+    /* 4:4:4 */
+    if (depth == 8)
+      profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN_444;
+    if (depth <= 10)
+      profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN_444_10;
+  } else if (chrome == 2) {
+    /* 4:2:2 */
+    profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN_422_10;
+  } else if (chrome == 1 || chrome == 0) {
+    /* 4:2:0 or 4:0:0 */
+    if (depth == 8)
+      profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN;
+    if (depth <= 10)
+      profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN10;
+    /* Always add STILL_PICTURE as a candidate. */
+    profile_candidates[num++] = GST_VAAPI_PROFILE_H265_MAIN_STILL_PICTURE;
+  }
+
+  if (num == 0) {
+    GST_ERROR ("Fail to find a profile for format %s.",
+        gst_video_format_to_string (format));
+    return FALSE;
+  }
 
-  /* Always start from "Main" profile for maximum
-     compatibility */
-  profile = GST_VAAPI_PROFILE_H265_MAIN;
-
-  if (format == GST_VIDEO_FORMAT_P010_10LE)
-    profile = GST_VAAPI_PROFILE_H265_MAIN10;
-  else if (format == GST_VIDEO_FORMAT_VUYA)
-    profile = GST_VAAPI_PROFILE_H265_MAIN_444;
-  else if (format == GST_VIDEO_FORMAT_Y410)
-    profile = GST_VAAPI_PROFILE_H265_MAIN_444_10;
-  else if (format == GST_VIDEO_FORMAT_Y210 || format == GST_VIDEO_FORMAT_YUY2)
-    profile = GST_VAAPI_PROFILE_H265_MAIN_422_10;
+  profile = GST_VAAPI_PROFILE_UNKNOWN;
+  for (i = 0; i < num; i++) {
+    if (!is_profile_allowed (encoder, profile_candidates[i]))
+      continue;
+    /* If we can get valid entrypoint, hw must support this profile. */
+    if (gst_vaapi_encoder_get_entrypoint (GST_VAAPI_ENCODER_CAST (encoder),
+            profile_candidates[i]) == GST_VAAPI_ENTRYPOINT_INVALID)
+      continue;
+
+    profile = profile_candidates[i];
+    break;
+  }
+
+  if (profile == GST_VAAPI_PROFILE_UNKNOWN) {
+    GST_ERROR ("Fail to find a supported profile %sfor format %s.",
+        GST_VAAPI_ENCODER_TUNE (encoder) == GST_VAAPI_ENCODER_TUNE_LOW_POWER ?
+        "in low power mode " : "", gst_video_format_to_string (format));
+    return FALSE;
+  }
 
   encoder->profile = profile;
   encoder->profile_idc = gst_vaapi_utils_h265_get_profile_idc (profile);
@@ -2263,16 +2252,13 @@ ensure_profile_tier_level (GstVaapiEncoderH265 * encoder)
   const GstVaapiTierH265 tier = encoder->tier;
   const GstVaapiLevelH265 level = encoder->level;
 
-  if (!ensure_profile (encoder) || !ensure_profile_limits (encoder))
+  if (!ensure_profile (encoder))
     return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 
   encoder->entrypoint =
       gst_vaapi_encoder_get_entrypoint (GST_VAAPI_ENCODER_CAST (encoder),
       encoder->profile);
-  if (encoder->entrypoint == GST_VAAPI_ENTRYPOINT_INVALID) {
-    GST_WARNING ("Cannot find valid entrypoint");
-    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
-  }
+  g_assert (encoder->entrypoint != GST_VAAPI_ENTRYPOINT_INVALID);
 
   /* Ensure bitrate if not set already and derive the right level to use */
   ensure_bitrate (encoder);
@@ -3096,8 +3082,7 @@ set_context_info (GstVaapiEncoder * base_encoder)
   base_encoder->codedbuf_size += encoder->num_slices * (4 +
       GST_ROUND_UP_8 (MAX_SLICE_HDR_SIZE + MAX_SHORT_TERM_REFPICSET_SIZE) / 8);
 
-  if (!ensure_hw_profile (encoder))
-    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
+  GST_VAAPI_ENCODER_CAST (encoder)->profile = encoder->profile;
 
   base_encoder->num_ref_frames = (encoder->num_ref_frames
       + (encoder->num_bframes > 0 ? 1 : 0) + DEFAULT_SURFACES_COUNT);