avdec: fix paletted formats

Pass the palette as the side data for AVPacket so that it is written in the
second plane of output AVFrame.

3 files changed, 19 insertions, 15 deletions

diff --git a/ext/ffmpeg/gstffmpegcodecmap.c b/ext/ffmpeg/gstffmpegcodecmap.c
index 47a2d71..4c2727a 100644
--- a/ext/ffmpeg/gstffmpegcodecmap.c
+++ b/ext/ffmpeg/gstffmpegcodecmap.c
@@ -51,6 +51,7 @@ gst_ffmpeg_get_palette (const GstCaps * caps, AVCodecContext * context)
   /* do we have a palette? */
   if ((palette_v = gst_structure_get_value (str, "palette_data")) && context) {
     palette = gst_value_get_buffer (palette_v);
+    GST_DEBUG ("got palette data %p", palette);
     if (gst_buffer_get_size (palette) >= AVPALETTE_SIZE) {
       if (context->palctrl)
         av_free (context->palctrl);
@@ -58,6 +59,7 @@ gst_ffmpeg_get_palette (const GstCaps * caps, AVCodecContext * context)
       context->palctrl->palette_changed = 1;
       gst_buffer_extract (palette, 0, context->palctrl->palette,
           AVPALETTE_SIZE);
+      GST_DEBUG ("extracted palette data");
     }
   }
 }
@@ -1710,7 +1712,7 @@ gst_ffmpeg_pixfmt_to_video_format (enum PixelFormat pix_fmt)
       fmt = GST_VIDEO_FORMAT_RGB15;
       break;
     case PIX_FMT_PAL8:
-      fmt = GST_VIDEO_FORMAT_RGB8_PALETTED;
+      fmt = GST_VIDEO_FORMAT_RGB8P;
       break;
     case PIX_FMT_GRAY8:
       fmt = GST_VIDEO_FORMAT_GRAY8;
@@ -2005,6 +2007,8 @@ gst_ffmpeg_caps_to_pixfmt (const GstCaps * caps,
         context->sample_aspect_ratio.num);
   }
 
+  gst_ffmpeg_get_palette (caps, context);
+
   if (!raw)
     return;
 
@@ -2063,9 +2067,8 @@ gst_ffmpeg_caps_to_pixfmt (const GstCaps * caps,
     case GST_VIDEO_FORMAT_RGB15:
       context->pix_fmt = PIX_FMT_RGB555;
       break;
-    case GST_VIDEO_FORMAT_RGB8_PALETTED:
+    case GST_VIDEO_FORMAT_RGB8P:
       context->pix_fmt = PIX_FMT_PAL8;
-      gst_ffmpeg_get_palette (caps, context);
       break;
     default:
       break;
@@ -2436,7 +2439,6 @@ gst_ffmpeg_caps_with_codecid (enum CodecID codec_id,
   switch (codec_type) {
     case AVMEDIA_TYPE_VIDEO:
       gst_ffmpeg_caps_to_pixfmt (caps, context, codec_id == CODEC_ID_RAWVIDEO);
-      gst_ffmpeg_get_palette (caps, context);
       break;
     case AVMEDIA_TYPE_AUDIO:
       gst_ffmpeg_caps_to_smpfmt (caps, context, FALSE);
diff --git a/ext/ffmpeg/gstffmpegdec.c b/ext/ffmpeg/gstffmpegdec.c
index 605ebdb..a15507b 100644
--- a/ext/ffmpeg/gstffmpegdec.c
+++ b/ext/ffmpeg/gstffmpegdec.c
@@ -1446,8 +1446,8 @@ gst_ffmpegdec_audio_negotiate (GstFFMpegDec * ffmpegdec, gboolean force)
   memcpy (ffmpegdec->format.audio.gst_layout,
       ffmpegdec->format.audio.ffmpeg_layout,
       sizeof (GstAudioChannelPosition) * ffmpegdec->format.audio.channels);
-  gst_audio_channel_positions_to_valid_order (ffmpegdec->format.
-      audio.gst_layout, ffmpegdec->format.audio.channels);
+  gst_audio_channel_positions_to_valid_order (ffmpegdec->format.audio.
+      gst_layout, ffmpegdec->format.audio.channels);
 
   GST_LOG_OBJECT (ffmpegdec, "output caps %" GST_PTR_FORMAT, caps);
 
@@ -1886,6 +1886,16 @@ gst_ffmpegdec_video_frame (GstFFMpegDec * ffmpegdec,
 
   /* now decode the frame */
   gst_avpacket_init (&packet, data, size);
+
+  if (ffmpegdec->context->palctrl) {
+    guint8 *pal;
+
+    pal = av_packet_new_side_data (&packet, AV_PKT_DATA_PALETTE,
+        AVPALETTE_SIZE);
+    memcpy (pal, ffmpegdec->context->palctrl->palette, AVPALETTE_SIZE);
+    GST_DEBUG_OBJECT (ffmpegdec, "copy pal %p %p", &packet, pal);
+  }
+
   len = avcodec_decode_video2 (ffmpegdec->context,
       ffmpegdec->picture, &have_data, &packet);
 
@@ -2124,14 +2134,6 @@ gst_ffmpegdec_video_frame (GstFFMpegDec * ffmpegdec,
   GST_LOG_OBJECT (ffmpegdec, "next out %" GST_TIME_FORMAT,
       GST_TIME_ARGS (ffmpegdec->next_out));
 
-  /* palette is not part of raw video frame in gst and the size
-   * of the outgoing buffer needs to be adjusted accordingly */
-  if (ffmpegdec->context->palctrl != NULL) {
-
-    gst_buffer_resize (*outbuf, 0,
-        gst_buffer_get_size (*outbuf) - AVPALETTE_SIZE);
-  }
-
   /* now see if we need to clip the buffer against the segment boundaries. */
   if (G_UNLIKELY (!clip_video_buffer (ffmpegdec, *outbuf, out_pts,
               out_duration)))
diff --git a/ext/libswscale/gstffmpegscale.c b/ext/libswscale/gstffmpegscale.c
index 63ffe14..31849ba 100644
--- a/ext/libswscale/gstffmpegscale.c
+++ b/ext/libswscale/gstffmpegscale.c
@@ -529,7 +529,7 @@ gst_ffmpeg_caps_to_pixfmt (const GstCaps * caps)
     case GST_VIDEO_FORMAT_RGB15:
       pix_fmt = PIX_FMT_RGB555;
       break;
-    case GST_VIDEO_FORMAT_RGB8_PALETTED:
+    case GST_VIDEO_FORMAT_RGB8P:
       pix_fmt = PIX_FMT_PAL8;
       break;
     default: