libav: avcodec_close() also resets fields like the AVCodec

We need to reload the defaults for the codec after closing it,
otherwise we can't access codec information like the supported
sample rates and can crash.

https://bugzilla.gnome.org/show_bug.cgi?id=707040

4 files changed, 158 insertions, 17 deletions

diff --git a/ext/libav/gstavauddec.c b/ext/libav/gstavauddec.c
index 6714185..627dc93 100644
--- a/ext/libav/gstavauddec.c
+++ b/ext/libav/gstavauddec.c
@@ -43,6 +43,7 @@ static void gst_ffmpegauddec_class_init (GstFFMpegAudDecClass * klass);
 static void gst_ffmpegauddec_init (GstFFMpegAudDec * ffmpegdec);
 static void gst_ffmpegauddec_finalize (GObject * object);
 
+static gboolean gst_ffmpegauddec_start (GstAudioDecoder * decoder);
 static gboolean gst_ffmpegauddec_stop (GstAudioDecoder * decoder);
 static void gst_ffmpegauddec_flush (GstAudioDecoder * decoder, gboolean hard);
 static gboolean gst_ffmpegauddec_set_format (GstAudioDecoder * decoder,
@@ -120,6 +121,7 @@ gst_ffmpegauddec_class_init (GstFFMpegAudDecClass * klass)
 
   gobject_class->finalize = gst_ffmpegauddec_finalize;
 
+  gstaudiodecoder_class->start = GST_DEBUG_FUNCPTR (gst_ffmpegauddec_start);
   gstaudiodecoder_class->stop = GST_DEBUG_FUNCPTR (gst_ffmpegauddec_stop);
   gstaudiodecoder_class->set_format =
       GST_DEBUG_FUNCPTR (gst_ffmpegauddec_set_format);
@@ -136,6 +138,7 @@ gst_ffmpegauddec_init (GstFFMpegAudDec * ffmpegdec)
 
   /* some ffmpeg data */
   ffmpegdec->context = avcodec_alloc_context3 (klass->in_plugin);
+  ffmpegdec->context->opaque = ffmpegdec;
   ffmpegdec->opened = FALSE;
 
   gst_audio_decoder_set_drainable (GST_AUDIO_DECODER (ffmpegdec), TRUE);
@@ -155,9 +158,13 @@ gst_ffmpegauddec_finalize (GObject * object)
 }
 
 /* With LOCK */
-static void
-gst_ffmpegauddec_close (GstFFMpegAudDec * ffmpegdec)
+static gboolean
+gst_ffmpegauddec_close (GstFFMpegAudDec * ffmpegdec, gboolean reset)
 {
+  GstFFMpegAudDecClass *oclass;
+
+  oclass = (GstFFMpegAudDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
+
   GST_LOG_OBJECT (ffmpegdec, "closing libav codec");
 
   gst_caps_replace (&ffmpegdec->last_caps, NULL);
@@ -170,6 +177,37 @@ gst_ffmpegauddec_close (GstFFMpegAudDec * ffmpegdec)
     av_free (ffmpegdec->context->extradata);
     ffmpegdec->context->extradata = NULL;
   }
+
+  if (reset) {
+    if (avcodec_get_context_defaults3 (ffmpegdec->context,
+            oclass->in_plugin) < 0) {
+      GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
+      return FALSE;
+    }
+    ffmpegdec->context->opaque = ffmpegdec;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_ffmpegauddec_start (GstAudioDecoder * decoder)
+{
+  GstFFMpegAudDec *ffmpegdec = (GstFFMpegAudDec *) decoder;
+  GstFFMpegAudDecClass *oclass;
+
+  oclass = (GstFFMpegAudDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
+
+  GST_OBJECT_LOCK (ffmpegdec);
+  if (avcodec_get_context_defaults3 (ffmpegdec->context, oclass->in_plugin) < 0) {
+    GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
+    GST_OBJECT_UNLOCK (ffmpegdec);
+    return FALSE;
+  }
+  ffmpegdec->context->opaque = ffmpegdec;
+  GST_OBJECT_UNLOCK (ffmpegdec);
+
+  return TRUE;
 }
 
 static gboolean
@@ -178,7 +216,7 @@ gst_ffmpegauddec_stop (GstAudioDecoder * decoder)
   GstFFMpegAudDec *ffmpegdec = (GstFFMpegAudDec *) decoder;
 
   GST_OBJECT_LOCK (ffmpegdec);
-  gst_ffmpegauddec_close (ffmpegdec);
+  gst_ffmpegauddec_close (ffmpegdec, FALSE);
   GST_OBJECT_UNLOCK (ffmpegdec);
   gst_audio_info_init (&ffmpegdec->info);
   gst_caps_replace (&ffmpegdec->last_caps, NULL);
@@ -209,7 +247,7 @@ gst_ffmpegauddec_open (GstFFMpegAudDec * ffmpegdec)
   /* ERRORS */
 could_not_open:
   {
-    gst_ffmpegauddec_close (ffmpegdec);
+    gst_ffmpegauddec_close (ffmpegdec, TRUE);
     GST_DEBUG_OBJECT (ffmpegdec, "avdec_%s: Failed to open libav codec",
         oclass->in_plugin->name);
     return FALSE;
@@ -292,7 +330,10 @@ gst_ffmpegauddec_set_format (GstAudioDecoder * decoder, GstCaps * caps)
     GST_OBJECT_UNLOCK (ffmpegdec);
     gst_ffmpegauddec_drain (ffmpegdec);
     GST_OBJECT_LOCK (ffmpegdec);
-    gst_ffmpegauddec_close (ffmpegdec);
+    if (!gst_ffmpegauddec_close (ffmpegdec, TRUE)) {
+      GST_OBJECT_UNLOCK (ffmpegdec);
+      return FALSE;
+    }
   }
 
   /* get size and so */
@@ -303,7 +344,6 @@ gst_ffmpegauddec_set_format (GstAudioDecoder * decoder, GstCaps * caps)
   ffmpegdec->context->workaround_bugs |= FF_BUG_AUTODETECT;
   ffmpegdec->context->err_recognition = 1;
 
-  ffmpegdec->context->opaque = ffmpegdec;
   ffmpegdec->context->get_buffer = gst_ffmpegauddec_get_buffer;
   ffmpegdec->context->reget_buffer = NULL;
   ffmpegdec->context->release_buffer = NULL;
diff --git a/ext/libav/gstavaudenc.c b/ext/libav/gstavaudenc.c
index bc07121..ee758fa 100644
--- a/ext/libav/gstavaudenc.c
+++ b/ext/libav/gstavaudenc.c
@@ -66,6 +66,7 @@ static gboolean gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder,
     GstAudioInfo * info);
 static GstFlowReturn gst_ffmpegaudenc_handle_frame (GstAudioEncoder * encoder,
     GstBuffer * inbuf);
+static gboolean gst_ffmpegaudenc_start (GstAudioEncoder * encoder);
 static gboolean gst_ffmpegaudenc_stop (GstAudioEncoder * encoder);
 static void gst_ffmpegaudenc_flush (GstAudioEncoder * encoder);
 
@@ -153,6 +154,7 @@ gst_ffmpegaudenc_class_init (GstFFMpegAudEncClass * klass)
 
   gobject_class->finalize = gst_ffmpegaudenc_finalize;
 
+  gstaudioencoder_class->start = GST_DEBUG_FUNCPTR (gst_ffmpegaudenc_start);
   gstaudioencoder_class->stop = GST_DEBUG_FUNCPTR (gst_ffmpegaudenc_stop);
   gstaudioencoder_class->getcaps = GST_DEBUG_FUNCPTR (gst_ffmpegaudenc_getcaps);
   gstaudioencoder_class->flush = GST_DEBUG_FUNCPTR (gst_ffmpegaudenc_flush);
@@ -187,6 +189,22 @@ gst_ffmpegaudenc_finalize (GObject * object)
 }
 
 static gboolean
+gst_ffmpegaudenc_start (GstAudioEncoder * encoder)
+{
+  GstFFMpegAudEnc *ffmpegaudenc = (GstFFMpegAudEnc *) encoder;
+  GstFFMpegAudEncClass *oclass =
+      (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc);
+
+  if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
+          oclass->in_plugin) < 0) {
+    GST_DEBUG_OBJECT (ffmpegaudenc, "Failed to set context defaults");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
 gst_ffmpegaudenc_stop (GstAudioEncoder * encoder)
 {
   GstFFMpegAudEnc *ffmpegaudenc = (GstFFMpegAudEnc *) encoder;
@@ -239,6 +257,11 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
   if (ffmpegaudenc->opened) {
     gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
     ffmpegaudenc->opened = FALSE;
+    if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
+            oclass->in_plugin) < 0) {
+      GST_DEBUG_OBJECT (ffmpegaudenc, "Failed to set context defaults");
+      return FALSE;
+    }
   }
 
   /* if we set it in _getcaps we should set it also in _link */
@@ -287,10 +310,12 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
 
   /* open codec */
   if (gst_ffmpeg_avcodec_open (ffmpegaudenc->context, oclass->in_plugin) < 0) {
-    if (ffmpegaudenc->context->priv_data)
-      gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
+    gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
     GST_DEBUG_OBJECT (ffmpegaudenc, "avenc_%s: Failed to open FFMPEG codec",
         oclass->in_plugin->name);
+    if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegaudenc, "Failed to set context defaults");
     return FALSE;
   }
 
@@ -316,6 +341,9 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
     gst_caps_unref (allowed_caps);
     gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
     GST_DEBUG ("Unsupported codec - no caps found");
+    if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegaudenc, "Failed to set context defaults");
     return FALSE;
   }
 
@@ -332,6 +360,9 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
           icaps)) {
     gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
     gst_caps_unref (icaps);
+    if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegaudenc, "Failed to set context defaults");
     return FALSE;
   }
   gst_caps_unref (icaps);
diff --git a/ext/libav/gstavviddec.c b/ext/libav/gstavviddec.c
index d5dc37d..52b03aa 100644
--- a/ext/libav/gstavviddec.c
+++ b/ext/libav/gstavviddec.c
@@ -68,6 +68,7 @@ static gboolean gst_ffmpegviddec_set_format (GstVideoDecoder * decoder,
     GstVideoCodecState * state);
 static GstFlowReturn gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
     GstVideoCodecFrame * frame);
+static gboolean gst_ffmpegviddec_start (GstVideoDecoder * decoder);
 static gboolean gst_ffmpegviddec_stop (GstVideoDecoder * decoder);
 static gboolean gst_ffmpegviddec_flush (GstVideoDecoder * decoder);
 static gboolean gst_ffmpegviddec_decide_allocation (GstVideoDecoder * decoder,
@@ -234,6 +235,7 @@ gst_ffmpegviddec_class_init (GstFFMpegVidDecClass * klass)
 
   viddec_class->set_format = gst_ffmpegviddec_set_format;
   viddec_class->handle_frame = gst_ffmpegviddec_handle_frame;
+  viddec_class->start = gst_ffmpegviddec_start;
   viddec_class->stop = gst_ffmpegviddec_stop;
   viddec_class->flush = gst_ffmpegviddec_flush;
   viddec_class->finish = gst_ffmpegviddec_finish;
@@ -275,15 +277,18 @@ gst_ffmpegviddec_finalize (GObject * object)
 
 
 /* with LOCK */
-static void
-gst_ffmpegviddec_close (GstFFMpegVidDec * ffmpegdec)
+static gboolean
+gst_ffmpegviddec_close (GstFFMpegVidDec * ffmpegdec, gboolean reset)
 {
+  GstFFMpegVidDecClass *oclass;
+
+  oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
+
   GST_LOG_OBJECT (ffmpegdec, "closing ffmpeg codec");
 
   gst_caps_replace (&ffmpegdec->last_caps, NULL);
 
-  if (ffmpegdec->context->priv_data)
-    gst_ffmpeg_avcodec_close (ffmpegdec->context);
+  gst_ffmpeg_avcodec_close (ffmpegdec->context);
   ffmpegdec->opened = FALSE;
 
   gst_buffer_replace (&ffmpegdec->palette, NULL);
@@ -292,6 +297,16 @@ gst_ffmpegviddec_close (GstFFMpegVidDec * ffmpegdec)
     av_free (ffmpegdec->context->extradata);
     ffmpegdec->context->extradata = NULL;
   }
+
+  if (reset) {
+    if (avcodec_get_context_defaults3 (ffmpegdec->context,
+            oclass->in_plugin) < 0) {
+      GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
+      return FALSE;
+    }
+    ffmpegdec->context->opaque = ffmpegdec;
+  }
+  return TRUE;
 }
 
 /* with LOCK */
@@ -328,7 +343,7 @@ gst_ffmpegviddec_open (GstFFMpegVidDec * ffmpegdec)
   /* ERRORS */
 could_not_open:
   {
-    gst_ffmpegviddec_close (ffmpegdec);
+    gst_ffmpegviddec_close (ffmpegdec, TRUE);
     GST_DEBUG_OBJECT (ffmpegdec, "avdec_%s: Failed to open libav codec",
         oclass->in_plugin->name);
     return FALSE;
@@ -384,7 +399,10 @@ gst_ffmpegviddec_set_format (GstVideoDecoder * decoder,
     GST_OBJECT_UNLOCK (ffmpegdec);
     gst_ffmpegviddec_drain (ffmpegdec);
     GST_OBJECT_LOCK (ffmpegdec);
-    gst_ffmpegviddec_close (ffmpegdec);
+    if (!gst_ffmpegviddec_close (ffmpegdec, TRUE)) {
+      GST_OBJECT_UNLOCK (ffmpegdec);
+      return FALSE;
+    }
   }
 
   gst_caps_replace (&ffmpegdec->last_caps, state->caps);
@@ -1404,13 +1422,34 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
   return ret;
 }
 
+
+static gboolean
+gst_ffmpegviddec_start (GstVideoDecoder * decoder)
+{
+  GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
+  GstFFMpegVidDecClass *oclass;
+
+  oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
+
+  GST_OBJECT_LOCK (ffmpegdec);
+  if (avcodec_get_context_defaults3 (ffmpegdec->context, oclass->in_plugin) < 0) {
+    GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
+    GST_OBJECT_UNLOCK (ffmpegdec);
+    return FALSE;
+  }
+  ffmpegdec->context->opaque = ffmpegdec;
+  GST_OBJECT_UNLOCK (ffmpegdec);
+
+  return TRUE;
+}
+
 static gboolean
 gst_ffmpegviddec_stop (GstVideoDecoder * decoder)
 {
   GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
 
   GST_OBJECT_LOCK (ffmpegdec);
-  gst_ffmpegviddec_close (ffmpegdec);
+  gst_ffmpegviddec_close (ffmpegdec, FALSE);
   GST_OBJECT_UNLOCK (ffmpegdec);
   g_free (ffmpegdec->padded);
   ffmpegdec->padded = NULL;
diff --git a/ext/libav/gstavvidenc.c b/ext/libav/gstavvidenc.c
index ca89156..83e052c 100644
--- a/ext/libav/gstavvidenc.c
+++ b/ext/libav/gstavvidenc.c
@@ -92,6 +92,7 @@ static void gst_ffmpegvidenc_base_init (GstFFMpegVidEncClass * klass);
 static void gst_ffmpegvidenc_init (GstFFMpegVidEnc * ffmpegenc);
 static void gst_ffmpegvidenc_finalize (GObject * object);
 
+static gboolean gst_ffmpegvidenc_start (GstVideoEncoder * encoder);
 static gboolean gst_ffmpegvidenc_stop (GstVideoEncoder * encoder);
 static GstFlowReturn gst_ffmpegvidenc_finish (GstVideoEncoder * encoder);
 static gboolean gst_ffmpegvidenc_set_format (GstVideoEncoder * encoder,
@@ -208,6 +209,7 @@ gst_ffmpegvidenc_class_init (GstFFMpegVidEncClass * klass)
   /* register additional properties, possibly dependent on the exact CODEC */
   gst_ffmpeg_cfg_install_property (klass, PROP_CFG_BASE);
 
+  venc_class->start = gst_ffmpegvidenc_start;
   venc_class->stop = gst_ffmpegvidenc_stop;
   venc_class->finish = gst_ffmpegvidenc_finish;
   venc_class->handle_frame = gst_ffmpegvidenc_handle_frame;
@@ -291,6 +293,11 @@ gst_ffmpegvidenc_set_format (GstVideoEncoder * encoder,
   if (ffmpegenc->opened) {
     gst_ffmpeg_avcodec_close (ffmpegenc->context);
     ffmpegenc->opened = FALSE;
+    if (avcodec_get_context_defaults3 (ffmpegenc->context,
+            oclass->in_plugin) < 0) {
+      GST_DEBUG_OBJECT (ffmpegenc, "Failed to set context defaults");
+      return FALSE;
+    }
   }
 
   /* if we set it in _getcaps we should set it also in _link */
@@ -475,8 +482,10 @@ file_read_err:
 
 open_codec_fail:
   {
-    if (ffmpegenc->context->priv_data)
-      gst_ffmpeg_avcodec_close (ffmpegenc->context);
+    gst_ffmpeg_avcodec_close (ffmpegenc->context);
+    if (avcodec_get_context_defaults3 (ffmpegenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegenc, "Failed to set context defaults");
     if (ffmpegenc->context->stats_in)
       g_free (ffmpegenc->context->stats_in);
     GST_DEBUG_OBJECT (ffmpegenc, "avenc_%s: Failed to open libav codec",
@@ -487,6 +496,9 @@ open_codec_fail:
 pix_fmt_err:
   {
     gst_ffmpeg_avcodec_close (ffmpegenc->context);
+    if (avcodec_get_context_defaults3 (ffmpegenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegenc, "Failed to set context defaults");
     GST_DEBUG_OBJECT (ffmpegenc,
         "avenc_%s: AV wants different colourspace (%d given, %d wanted)",
         oclass->in_plugin->name, pix_fmt, ffmpegenc->context->pix_fmt);
@@ -503,6 +515,9 @@ bad_input_fmt:
 unsupported_codec:
   {
     gst_ffmpeg_avcodec_close (ffmpegenc->context);
+    if (avcodec_get_context_defaults3 (ffmpegenc->context,
+            oclass->in_plugin) < 0)
+      GST_DEBUG_OBJECT (ffmpegenc, "Failed to set context defaults");
     GST_DEBUG ("Unsupported codec - no caps found");
     return FALSE;
   }
@@ -798,6 +813,22 @@ gst_ffmpegvidenc_flush (GstVideoEncoder * encoder)
 }
 
 static gboolean
+gst_ffmpegvidenc_start (GstVideoEncoder * encoder)
+{
+  GstFFMpegVidEnc *ffmpegenc = (GstFFMpegVidEnc *) encoder;
+  GstFFMpegVidEncClass *oclass =
+      (GstFFMpegVidEncClass *) G_OBJECT_GET_CLASS (ffmpegenc);
+
+  /* close old session */
+  if (avcodec_get_context_defaults3 (ffmpegenc->context, oclass->in_plugin) < 0) {
+    GST_DEBUG_OBJECT (ffmpegenc, "Failed to set context defaults");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
 gst_ffmpegvidenc_stop (GstVideoEncoder * encoder)
 {
   GstFFMpegVidEnc *ffmpegenc = (GstFFMpegVidEnc *) encoder;