diff options
| -rw-r--r-- | gst/alpha/Makefile.am | 12 | ||||
| -rw-r--r-- | gst/alpha/gstalphamask.c | 1138 | ||||
| -rw-r--r-- | gst/alpha/gstalphamask.h | 90 | ||||
| -rw-r--r-- | gst/alpha/meson.build | 8 |
4 files changed, 1246 insertions, 2 deletions
diff --git a/gst/alpha/Makefile.am b/gst/alpha/Makefile.am index abadffe59..8fff1aeaa 100644 --- a/gst/alpha/Makefile.am +++ b/gst/alpha/Makefile.am @@ -1,4 +1,4 @@ -plugin_LTLIBRARIES = libgstalpha.la libgstalphacolor.la +plugin_LTLIBRARIES = libgstalpha.la libgstalphacolor.la libgstalphamask.la libgstalpha_la_SOURCES = gstalpha.c libgstalpha_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) \ @@ -16,4 +16,12 @@ libgstalphacolor_la_LIBADD = $(GST_PLUGINS_BASE_LIBS) -lgstvideo-$(GST_API_VERSI libgstalphacolor_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) libgstalphacolor_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) -noinst_HEADERS = gstalpha.h gstalphacolor.h +libgstalphamask_la_SOURCES = gstalphamask.c +libgstalphamask_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) \ + $(GST_BASE_CFLAGS) $(GST_CFLAGS) +libgstalphamask_la_LIBADD = $(GST_PLUGINS_BASE_LIBS) -lgstvideo-$(GST_API_VERSION) \ + $(GST_BASE_LIBS) $(GST_LIBS) +libgstalphamask_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstalphamask_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = gstalpha.h gstalphacolor.h gstalphamask.h diff --git a/gst/alpha/gstalphamask.c b/gst/alpha/gstalphamask.c new file mode 100644 index 000000000..bbb52de2f --- /dev/null +++ b/gst/alpha/gstalphamask.c @@ -0,0 +1,1138 @@ +/* GStreamer + * Copyright (C) 2016 Oblong Industries + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-alphamask + * + * The alphamask element combines a video and an alpha stream to produce + * transparent videos in A420, ARGB or AYUV formats. The alpha channel is + * made by reinterpreting a GRAY8 auxiliary video stream as an alpha mask. + * + * Sample pipeline: + * |[ + * gst-launch-1.0 videotestsrc pattern=2 ! queue ! mixer.sink_0 \ + * videotestsrc pattern=18 ! queue ! am.alpha_sink \ + * videotestsrc ! queue ! alphamask name=am ! queue ! mixer.sink_1 \ + * videomixer name=mixer sink_0::zorder=0 sink_1::zorder=1 ! queue ! \ + * glimagesink + * ]| + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include "gstalphamask.h" +#include <string.h> /* for memcpy */ + +GST_DEBUG_CATEGORY_STATIC (alphamask_debug); +#define GST_CAT_DEFAULT alphamask_debug + +#define FORMATS " { AYUV, A420, I420, YV12, NV12, NV21, BGRA, ARGB, RGBA, "\ + " ABGR, Y444, Y42B, YUY2, UYVY, YVYU, Y41B, RGB, BGR, "\ + " xRGB, xBGR, RGBx, BGRx } " + +static GstStaticPadTemplate vsink_factory = +GST_STATIC_PAD_TEMPLATE ("video_sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) + ); + +static GstStaticPadTemplate asink_factory = +GST_STATIC_PAD_TEMPLATE ("alpha_sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ GRAY8 }")) + ); + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ A420, ARGB, AYUV }")) + ); + +#define DEFAULT_FORMAT GST_VIDEO_FORMAT_A420 + +#define gst_alpha_mask_parent_class parent_class +G_DEFINE_TYPE (GstAlphaMask, gst_alpha_mask, GST_TYPE_ELEMENT); + +#define GST_ALPHA_MASK_GET_LOCK(o) (&GST_ALPHA_MASK (o)->lock) +#define GST_ALPHA_MASK_GET_COND(o) (&GST_ALPHA_MASK (o)->cond) +#define GST_ALPHA_MASK_LOCK(o) (g_mutex_lock (GST_ALPHA_MASK_GET_LOCK (o))) +#define GST_ALPHA_MASK_UNLOCK(o) (g_mutex_unlock (GST_ALPHA_MASK_GET_LOCK (o))) +#define GST_ALPHA_MASK_WAIT(o) (g_cond_wait (GST_ALPHA_MASK_GET_COND (o), GST_ALPHA_MASK_GET_LOCK (o))) +#define GST_ALPHA_MASK_SIGNAL(o) (g_cond_signal (GST_ALPHA_MASK_GET_COND (o))) +#define GST_ALPHA_MASK_BROADCAST(o)(g_cond_broadcast (GST_ALPHA_MASK_GET_COND (o))) + +static void +copy_alpha_packed_u1 (guint8 * dst, guint dstride, guint8 * src, guint sstride, + guint width, guint height) +{ + guint i, j; + + for (i = 0; i < height; i++) { + for (j = 0; j < width; j++) + dst[j * 4] = src[j]; + + dst += dstride; + src += sstride; + } +} + +static void +copy_alpha_packed_u4 (guint8 * dst, guint dstride, guint8 * src, guint sstride, + guint width, guint height) +{ + guint i, j; + guint w = width >> 2; + + for (i = 0; i < height; i++) { + guint32 *sd = (guint32 *) src; + guint8 *dd = dst; + for (j = 0; j < w; j++) { + guint32 c = sd[j]; + dd[0] = c & 0xff; + c >>= 8; + dd[4] = c & 0xff; + c >>= 8; + dd[8] = c & 0xff; + c >>= 8; + dd[12] = c & 0xff; + dd += 16; + } + + dst += dstride; + src += sstride; + } +} + +static void +copy_alpha_packed_u8 (guint8 * dst, guint dstride, guint8 * src, guint sstride, + guint width, guint height) +{ + guint i, j; + guint w = width >> 3; + + for (i = 0; i < height; i++) { + guint64 *sd = (guint64 *) src; + guint8 *dd = dst; + for (j = 0; j < w; j++) { + guint64 c = sd[j]; + dd[0] = c & 0xff; + c >>= 8; + dd[4] = c & 0xff; + c >>= 8; + dd[8] = c & 0xff; + c >>= 8; + dd[12] = c & 0xff; + c >>= 8; + dd[16] = c & 0xff; + c >>= 8; + dd[20] = c & 0xff; + c >>= 8; + dd[24] = c & 0xff; + c >>= 8; + dd[28] = c & 0xff; + dd += 32; + } + + dst += dstride; + src += sstride; + } +} + +static void +copy_alpha_packed (GstVideoFrame * aframe, GstVideoFrame * oframe) +{ + const GstVideoInfo *ainfo; + const GstVideoInfo *oinfo; + guint8 *sp, *dp; + guint w, h; + guint ss, ds; + + ainfo = &aframe->info; + oinfo = &oframe->info; + + sp = aframe->data[0]; + dp = oframe->data[0]; + + w = GST_VIDEO_FRAME_COMP_WIDTH (aframe, 0); + h = GST_VIDEO_FRAME_COMP_HEIGHT (aframe, 0); + + ss = GST_VIDEO_INFO_PLANE_STRIDE (ainfo, 0); + ds = GST_VIDEO_INFO_PLANE_STRIDE (oinfo, 0); + + if (w & 3) + copy_alpha_packed_u1 (dp, ds, sp, ss, w, h); + else if (w & 7) + copy_alpha_packed_u4 (dp, ds, sp, ss, w, h); + else + copy_alpha_packed_u8 (dp, ds, sp, ss, w, h); +} + +static void +copy_alpha_planar (GstVideoFrame * aframe, GstVideoFrame * oframe, guint plane) +{ + const GstVideoInfo *ainfo; + const GstVideoInfo *oinfo; + guint8 *sp, *dp; + guint w, h; + guint ss, ds; + + ainfo = &aframe->info; + oinfo = &oframe->info; + + sp = aframe->data[0]; + dp = oframe->data[plane]; + + w = GST_VIDEO_FRAME_COMP_WIDTH (aframe, 0); + h = GST_VIDEO_FRAME_COMP_HEIGHT (aframe, 0); + + ss = GST_VIDEO_INFO_PLANE_STRIDE (ainfo, 0); + ds = GST_VIDEO_INFO_PLANE_STRIDE (oinfo, plane); + + if (ss == ds) { + memcpy (dp, sp, h * ss); + } else { + guint j; + for (j = 0; j < h; j++) { + memcpy (dp, sp, w); + dp += ds; + sp += ss; + } + } +} + +static GstBuffer * +gst_alpha_mask_convert (GstAlphaMask * thiz, GstBuffer * ibuf) +{ + GstVideoFrame aframe, iframe, oframe; + GstBuffer *obuf; + static GstAllocationParams params = { 0, 15, 0, 0, }; + gint size = GST_VIDEO_INFO_SIZE (&thiz->oinfo); + + obuf = gst_buffer_new_allocate (NULL, size, ¶ms); + gst_buffer_copy_into (obuf, ibuf, GST_BUFFER_COPY_FLAGS | + GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + + /* Convert the frame into output format */ + if (!gst_video_frame_map (&iframe, &thiz->iinfo, ibuf, GST_MAP_READWRITE)) + goto invalid_in_frame; + + if (!gst_video_frame_map (&oframe, &thiz->oinfo, obuf, GST_MAP_READWRITE)) + goto invalid_out_frame; + + gst_video_converter_frame (thiz->convert, &iframe, &oframe); + gst_video_frame_unmap (&iframe); + gst_buffer_unref (ibuf); + + if (!thiz->alpha_buffer) + goto beach; + + if (!gst_video_frame_map (&aframe, &thiz->ainfo, thiz->alpha_buffer, + GST_MAP_READWRITE)) { + GST_DEBUG_OBJECT (thiz, "received invalid buffer"); + } else { + if (thiz->oformat == GST_VIDEO_FORMAT_A420) + copy_alpha_planar (&aframe, &oframe, 3); + else + copy_alpha_packed (&aframe, &oframe); + gst_video_frame_unmap (&aframe); + } + +beach: + gst_video_frame_unmap (&oframe); + + return obuf; + + /* ERRORS */ +invalid_in_frame: + { + gst_buffer_unref (ibuf); + GST_DEBUG_OBJECT (thiz, "received invalid buffer"); + return NULL; + } + +invalid_out_frame: + { + gst_video_frame_unmap (&iframe); + gst_buffer_unref (ibuf); + gst_buffer_unref (obuf); + GST_DEBUG_OBJECT (thiz, "invalid output buffer"); + return NULL; + } +} + +static gboolean +gst_alpha_mask_negotiate (GstAlphaMask * thiz, GstCaps * caps) +{ + GstCaps *output_caps = NULL; + GstCaps *template_caps; + GstCaps *allowed_caps = NULL; + GstVideoFormat format = DEFAULT_FORMAT; + GstVideoInfo info; + gboolean ret; + + GST_DEBUG_OBJECT (thiz, "performing negotiation"); + + /* Clear any pending reconfigure to avoid negotiating twice */ + gst_pad_check_reconfigure (thiz->srcpad); + + if (!caps || gst_caps_is_empty (caps)) + return FALSE; + + template_caps = gst_static_pad_template_get_caps (&src_factory); + allowed_caps = gst_pad_get_allowed_caps (thiz->srcpad); + + /* If downstream has ANY caps just go for our preferred format */ + if (allowed_caps == template_caps) { + GST_INFO_OBJECT (thiz, "downstream has ANY caps"); + } else if (allowed_caps) { + if (gst_caps_is_empty (allowed_caps)) { + gst_caps_unref (allowed_caps); + gst_caps_unref (template_caps); + return FALSE; + } + + allowed_caps = gst_caps_make_writable (allowed_caps); + allowed_caps = gst_caps_fixate (allowed_caps); + gst_video_info_from_caps (&info, allowed_caps); + format = GST_VIDEO_INFO_FORMAT (&info); + } + gst_caps_unref (allowed_caps); + gst_caps_unref (template_caps); + + gst_video_info_set_format (&info, format, thiz->width, thiz->height); + info.par_n = thiz->iinfo.par_n; + info.par_d = thiz->iinfo.par_d; + info.fps_n = thiz->iinfo.fps_n; + info.fps_d = thiz->iinfo.fps_d; + + GST_DEBUG_OBJECT (thiz, "Converting video from %d to %d", + GST_VIDEO_INFO_FORMAT (&thiz->iinfo), GST_VIDEO_INFO_FORMAT (&info)); + + if (thiz->convert) + gst_video_converter_free (thiz->convert); + + thiz->convert = gst_video_converter_new (&thiz->iinfo, &info, NULL); + if (!thiz->convert) { + GST_ERROR_OBJECT (thiz, "Video cannot be converted"); + return FALSE; + } + + thiz->oinfo = info; + thiz->oformat = format; + output_caps = gst_video_info_to_caps (&info); + + GST_DEBUG_OBJECT (thiz, "output video caps %" GST_PTR_FORMAT, output_caps); + ret = gst_pad_set_caps (thiz->srcpad, output_caps); + if (!ret) { + GST_DEBUG_OBJECT (thiz, "negotiation failed, schedule reconfigure"); + gst_pad_mark_reconfigure (thiz->srcpad); + } + + gst_caps_unref (output_caps); + + return ret; +} + +static GstFlowReturn +gst_alpha_mask_push_frame (GstAlphaMask * thiz, GstBuffer * ibuffer) +{ + GstBuffer *obuffer = NULL; + GstFlowReturn ret = GST_FLOW_OK; + + switch (thiz->oformat) { + case GST_VIDEO_FORMAT_A420: + if (thiz->iformat == GST_VIDEO_FORMAT_I420) { + GstMemory *mem = gst_buffer_peek_memory (thiz->alpha_buffer, 0); + obuffer = gst_buffer_make_writable (ibuffer); + gst_buffer_append_memory (obuffer, gst_memory_ref (mem)); + break; + } + case GST_VIDEO_FORMAT_ARGB: + case GST_VIDEO_FORMAT_AYUV: + obuffer = gst_alpha_mask_convert (thiz, ibuffer); + break; + default: + break; + } + + if (obuffer) + ret = gst_pad_push (thiz->srcpad, obuffer); + + return ret; +} + +/* Called with lock held */ +static void +gst_alpha_mask_pop_alpha (GstAlphaMask * thiz) +{ + g_return_if_fail (GST_IS_ALPHA_MASK (thiz)); + + if (thiz->alpha_buffer) { + GST_DEBUG_OBJECT (thiz, "releasing alpha buffer %p", thiz->alpha_buffer); + gst_buffer_unref (thiz->alpha_buffer); + thiz->alpha_buffer = NULL; + } + + /* Let the task know we used that buffer */ + GST_ALPHA_MASK_BROADCAST (thiz); +} + +static GstFlowReturn +gst_alpha_mask_video_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstAlphaMask *thiz; + GstFlowReturn ret = GST_FLOW_OK; + gboolean in_seg = FALSE; + guint64 start, stop, clip_start = 0, clip_stop = 0; + + thiz = GST_ALPHA_MASK (parent); + + if (!GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + goto missing_timestamp; + + /* ignore buffers that are outside of the current segment */ + start = GST_BUFFER_TIMESTAMP (buffer); + + if (!GST_BUFFER_DURATION_IS_VALID (buffer)) { + stop = GST_CLOCK_TIME_NONE; + } else { + stop = start + GST_BUFFER_DURATION (buffer); + } + + GST_LOG_OBJECT (thiz, "%" GST_SEGMENT_FORMAT " BUFFER: ts=%" + GST_TIME_FORMAT ", end=%" GST_TIME_FORMAT, &thiz->segment, + GST_TIME_ARGS (start), GST_TIME_ARGS (stop)); + + /* segment_clip() will adjust start unconditionally to segment_start if + * no stop time is provided, so handle this ourselves */ + if (stop == GST_CLOCK_TIME_NONE && start < thiz->segment.start) + goto out_of_segment; + + in_seg = gst_segment_clip (&thiz->segment, GST_FORMAT_TIME, start, stop, + &clip_start, &clip_stop); + + if (!in_seg) + goto out_of_segment; + + /* if the buffer is only partially in the segment, fix up stamps */ + if (clip_start != start || (stop != -1 && clip_stop != stop)) { + GST_DEBUG_OBJECT (thiz, "clipping buffer timestamp/duration to segment"); + buffer = gst_buffer_make_writable (buffer); + GST_BUFFER_TIMESTAMP (buffer) = clip_start; + if (stop != -1) + GST_BUFFER_DURATION (buffer) = clip_stop - clip_start; + } + + /* now, after we've done the clipping, fix up end time if there's no + * duration (we only use those estimated values internally though, we + * don't want to set bogus values on the buffer itself) */ + if (stop == -1) { + if (thiz->iinfo.fps_n && thiz->iinfo.fps_d) { + GST_DEBUG_OBJECT (thiz, "estimating duration based on framerate"); + stop = start + gst_util_uint64_scale_int (GST_SECOND, + thiz->iinfo.fps_d, thiz->iinfo.fps_n); + } else { + GST_LOG_OBJECT (thiz, "no duration, assuming minimal duration"); + stop = start + 1; /* we need to assume some interval */ + } + } + + gst_object_sync_values (GST_OBJECT (thiz), GST_BUFFER_TIMESTAMP (buffer)); + +wait_for_alpha_buf: + + GST_ALPHA_MASK_LOCK (thiz); + + if (thiz->video_flushing) + goto flushing; + + if (thiz->video_eos) + goto have_eos; + + /* Alpha pad not linked, just pass-trough */ + if (!thiz->alpha_linked) { + GST_ALPHA_MASK_UNLOCK (thiz); + /* Push the video frame */ + ret = gst_alpha_mask_push_frame (thiz, buffer); + } else { + /* Alpha pad linked, check if we have a alpha buffer queued */ + if (thiz->alpha_buffer) { + gboolean pop_alpha = FALSE, valid_alpha_time = TRUE; + GstClockTime alpha_start = GST_CLOCK_TIME_NONE; + GstClockTime alpha_end = GST_CLOCK_TIME_NONE; + GstClockTime alpha_running_time = GST_CLOCK_TIME_NONE; + GstClockTime alpha_running_time_end = GST_CLOCK_TIME_NONE; + GstClockTime vid_running_time, vid_running_time_end; + + /* if the alpha buffer isn't stamped right, pop it off the + * queue and display it for the current video frame only */ + if (!GST_BUFFER_TIMESTAMP_IS_VALID (thiz->alpha_buffer) || + !GST_BUFFER_DURATION_IS_VALID (thiz->alpha_buffer)) { + GST_WARNING_OBJECT (thiz, + "Got alpha buffer with invalid timestamp or duration"); + pop_alpha = TRUE; + valid_alpha_time = FALSE; + } else { + alpha_start = GST_BUFFER_TIMESTAMP (thiz->alpha_buffer); + alpha_end = alpha_start + GST_BUFFER_DURATION (thiz->alpha_buffer); + } + + vid_running_time = + gst_segment_to_running_time (&thiz->segment, GST_FORMAT_TIME, start); + vid_running_time_end = + gst_segment_to_running_time (&thiz->segment, GST_FORMAT_TIME, stop); + + /* If timestamp and duration are valid */ + if (valid_alpha_time) { + alpha_running_time = + gst_segment_to_running_time (&thiz->alpha_segment, + GST_FORMAT_TIME, alpha_start); + alpha_running_time_end = + gst_segment_to_running_time (&thiz->alpha_segment, + GST_FORMAT_TIME, alpha_end); + } + + GST_LOG_OBJECT (thiz, "T: %" GST_TIME_FORMAT " - %" GST_TIME_FORMAT, + GST_TIME_ARGS (alpha_running_time), + GST_TIME_ARGS (alpha_running_time_end)); + GST_LOG_OBJECT (thiz, "V: %" GST_TIME_FORMAT " - %" GST_TIME_FORMAT, + GST_TIME_ARGS (vid_running_time), + GST_TIME_ARGS (vid_running_time_end)); + + /* Alpha too old or in the future */ + if (valid_alpha_time && alpha_running_time_end <= vid_running_time) { + /* alpha buffer too old, get rid of it and do nothing */ + GST_LOG_OBJECT (thiz, "alpha buffer too old, popping"); + pop_alpha = FALSE; + gst_alpha_mask_pop_alpha (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + goto wait_for_alpha_buf; + } else if (valid_alpha_time && vid_running_time_end <= alpha_running_time) { + GST_LOG_OBJECT (thiz, "alpha in future, pushing video buffer"); + GST_ALPHA_MASK_UNLOCK (thiz); + /* Push the video frame */ + ret = gst_alpha_mask_push_frame (thiz, buffer); + } else { + GST_ALPHA_MASK_UNLOCK (thiz); + ret = gst_alpha_mask_push_frame (thiz, buffer); + + if (valid_alpha_time && alpha_running_time_end <= vid_running_time_end) { + GST_LOG_OBJECT (thiz, "alpha buffer not needed any longer"); + pop_alpha = TRUE; + } + } + if (pop_alpha) { + GST_ALPHA_MASK_LOCK (thiz); + gst_alpha_mask_pop_alpha (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + } + } else { + gboolean wait_for_alpha_buf = TRUE; + + if (thiz->alpha_eos) + wait_for_alpha_buf = FALSE; + + /* Alpha pad linked, but no alpha buffer available - what now? */ + if (thiz->alpha_segment.format == GST_FORMAT_TIME) { + GstClockTime alpha_start_running_time, alpha_position_running_time; + GstClockTime vid_running_time; + + vid_running_time = + gst_segment_to_running_time (&thiz->segment, GST_FORMAT_TIME, + GST_BUFFER_TIMESTAMP (buffer)); + alpha_start_running_time = + gst_segment_to_running_time (&thiz->alpha_segment, + GST_FORMAT_TIME, thiz->alpha_segment.start); + alpha_position_running_time = + gst_segment_to_running_time (&thiz->alpha_segment, + GST_FORMAT_TIME, thiz->alpha_segment.position); + + if ((GST_CLOCK_TIME_IS_VALID (alpha_start_running_time) && + vid_running_time < alpha_start_running_time) || + (GST_CLOCK_TIME_IS_VALID (alpha_position_running_time) && + vid_running_time < alpha_position_running_time)) { + wait_for_alpha_buf = FALSE; + } + } + + if (wait_for_alpha_buf) { + GST_DEBUG_OBJECT (thiz, "no alpha buffer, need to wait for one"); + GST_ALPHA_MASK_WAIT (thiz); + GST_DEBUG_OBJECT (thiz, "resuming"); + GST_ALPHA_MASK_UNLOCK (thiz); + goto wait_for_alpha_buf; + } else { + GST_ALPHA_MASK_UNLOCK (thiz); + GST_LOG_OBJECT (thiz, "no need to wait for a alpha buffer"); + ret = gst_pad_push (thiz->srcpad, buffer); + } + } + } + + /* Update position */ + thiz->segment.position = clip_start; + + return ret; + +missing_timestamp: + { + GST_WARNING_OBJECT (thiz, "buffer without timestamp, discarding"); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } + +flushing: + { + GST_ALPHA_MASK_UNLOCK (thiz); + GST_DEBUG_OBJECT (thiz, "flushing, discarding buffer"); + gst_buffer_unref (buffer); + return GST_FLOW_FLUSHING; + } +have_eos: + { + GST_ALPHA_MASK_UNLOCK (thiz); + GST_DEBUG_OBJECT (thiz, "eos, discarding buffer"); + gst_buffer_unref (buffer); + return GST_FLOW_EOS; + } +out_of_segment: + { + GST_DEBUG_OBJECT (thiz, "buffer out of segment, discarding"); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } +} + +static gboolean +gst_alpha_mask_video_setcaps (GstAlphaMask * thiz, GstCaps * caps) +{ + GstVideoInfo info; + gboolean ret = FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + GST_DEBUG_OBJECT (thiz, "received video caps %" GST_PTR_FORMAT, caps); + + thiz->iinfo = info; + thiz->width = GST_VIDEO_INFO_WIDTH (&info); + thiz->height = GST_VIDEO_INFO_HEIGHT (&info); + thiz->iformat = GST_VIDEO_INFO_FORMAT (&info); + + ret = gst_alpha_mask_negotiate (thiz, caps); + + return ret; + + /* ERRORS */ +invalid_caps: + { + GST_DEBUG_OBJECT (thiz, "could not parse caps"); + return FALSE; + } +} + +static gboolean +gst_alpha_mask_video_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = FALSE; + GstAlphaMask *thiz = NULL; + + thiz = GST_ALPHA_MASK (parent); + + GST_DEBUG_OBJECT (pad, "received event %s", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + ret = gst_alpha_mask_video_setcaps (thiz, caps); + gst_event_unref (event); + break; + } + case GST_EVENT_SEGMENT: + { + const GstSegment *segment; + + GST_DEBUG_OBJECT (thiz, "received new segment"); + + gst_event_parse_segment (event, &segment); + + if (segment->format == GST_FORMAT_TIME) { + GST_DEBUG_OBJECT (thiz, "VIDEO SEGMENT now: %" GST_SEGMENT_FORMAT, + &thiz->segment); + + gst_segment_copy_into (segment, &thiz->segment); + } else { + GST_ELEMENT_WARNING (thiz, STREAM, MUX, (NULL), + ("received non-TIME newsegment event on video input")); + } + + ret = gst_pad_event_default (pad, parent, event); + break; + } + case GST_EVENT_EOS: + GST_ALPHA_MASK_LOCK (thiz); + GST_INFO_OBJECT (thiz, "video EOS"); + thiz->video_eos = TRUE; + GST_ALPHA_MASK_UNLOCK (thiz); + ret = gst_pad_event_default (pad, parent, event); + break; + case GST_EVENT_FLUSH_START: + GST_ALPHA_MASK_LOCK (thiz); + GST_INFO_OBJECT (thiz, "video flush start"); + thiz->video_flushing = TRUE; + GST_ALPHA_MASK_BROADCAST (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + ret = gst_pad_event_default (pad, parent, event); + break; + case GST_EVENT_FLUSH_STOP: + GST_ALPHA_MASK_LOCK (thiz); + GST_INFO_OBJECT (thiz, "video flush stop"); + thiz->video_flushing = FALSE; + thiz->video_eos = FALSE; + gst_segment_init (&thiz->segment, GST_FORMAT_TIME); + GST_ALPHA_MASK_UNLOCK (thiz); + ret = gst_pad_event_default (pad, parent, event); + break; + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + + return ret; +} + +/* We receive alpha buffers here. If they are out of segment we just ignore them. + If the buffer is in our segment we keep it internally except if another one + is already waiting here, in that case we wait that it gets kicked out */ +static GstFlowReturn +gst_alpha_mask_alpha_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstAlphaMask *thiz = NULL; + gboolean in_seg = FALSE; + guint64 clip_start = 0, clip_stop = 0; + + thiz = GST_ALPHA_MASK (parent); + + GST_ALPHA_MASK_LOCK (thiz); + + if (thiz->alpha_flushing) { + GST_ALPHA_MASK_UNLOCK (thiz); + ret = GST_FLOW_FLUSHING; + GST_LOG_OBJECT (thiz, "alpha flushing"); + goto beach; + } + + if (thiz->alpha_eos) { + GST_ALPHA_MASK_UNLOCK (thiz); + ret = GST_FLOW_EOS; + GST_LOG_OBJECT (thiz, "alpha EOS"); + goto beach; + } + + GST_LOG_OBJECT (thiz, "%" GST_SEGMENT_FORMAT " BUFFER: ts=%" + GST_TIME_FORMAT ", end=%" GST_TIME_FORMAT, &thiz->segment, + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer) + + GST_BUFFER_DURATION (buffer))); + + if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (buffer))) { + GstClockTime stop; + + if (G_LIKELY (GST_BUFFER_DURATION_IS_VALID (buffer))) + stop = GST_BUFFER_TIMESTAMP (buffer) + GST_BUFFER_DURATION (buffer); + else + stop = GST_CLOCK_TIME_NONE; + + in_seg = gst_segment_clip (&thiz->alpha_segment, GST_FORMAT_TIME, + GST_BUFFER_TIMESTAMP (buffer), stop, &clip_start, &clip_stop); + } else { + in_seg = TRUE; + } + + if (in_seg) { + if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + GST_BUFFER_TIMESTAMP (buffer) = clip_start; + else if (GST_BUFFER_DURATION_IS_VALID (buffer)) + GST_BUFFER_DURATION (buffer) = clip_stop - clip_start; + + /* Wait for the previous buffer to go away */ + while (thiz->alpha_buffer != NULL) { + GST_DEBUG ("Pad %s:%s has a buffer queued, waiting", + GST_DEBUG_PAD_NAME (pad)); + GST_ALPHA_MASK_WAIT (thiz); + GST_DEBUG ("Pad %s:%s resuming", GST_DEBUG_PAD_NAME (pad)); + if (thiz->alpha_flushing) { + GST_ALPHA_MASK_UNLOCK (thiz); + ret = GST_FLOW_FLUSHING; + goto beach; + } + } + + if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + thiz->alpha_segment.position = clip_start; + + thiz->alpha_buffer = buffer; /* pass ownership of @buffer */ + buffer = NULL; + + /* in case the video chain is waiting for a alpha buffer, wake it up */ + GST_ALPHA_MASK_BROADCAST (thiz); + } + + GST_ALPHA_MASK_UNLOCK (thiz); + +beach: + if (buffer) + gst_buffer_unref (buffer); + + return ret; +} + +static gboolean +gst_alpha_mask_alpha_setcaps (GstAlphaMask * thiz, GstCaps * caps) +{ + GstVideoInfo info; + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + GST_DEBUG_OBJECT (thiz, "received alpha caps %" GST_PTR_FORMAT, caps); + thiz->ainfo = info; + + return TRUE; + + /* ERRORS */ +invalid_caps: + { + GST_DEBUG_OBJECT (thiz, "could not parse caps"); + return FALSE; + } +} + +static gboolean +gst_alpha_mask_alpha_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = FALSE; + GstAlphaMask *thiz = NULL; + + thiz = GST_ALPHA_MASK (parent); + + GST_LOG_OBJECT (pad, "received event %s", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + ret = gst_alpha_mask_alpha_setcaps (thiz, caps); + gst_event_unref (event); + break; + } + case GST_EVENT_SEGMENT: + { + const GstSegment *segment; + + thiz->alpha_eos = FALSE; + + gst_event_parse_segment (event, &segment); + + if (segment->format == GST_FORMAT_TIME) { + GST_ALPHA_MASK_LOCK (thiz); + gst_segment_copy_into (segment, &thiz->alpha_segment); + GST_DEBUG_OBJECT (thiz, "ALPHA SEGMENT now: %" GST_SEGMENT_FORMAT, + &thiz->alpha_segment); + GST_ALPHA_MASK_UNLOCK (thiz); + } else { + GST_ELEMENT_WARNING (thiz, STREAM, MUX, (NULL), + ("received non-TIME newsegment event on alpha input")); + } + + gst_event_unref (event); + ret = TRUE; + + /* wake up the video chain, it might be waiting for a alpha buffer or + * a alpha segment update */ + GST_ALPHA_MASK_LOCK (thiz); + GST_ALPHA_MASK_BROADCAST (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + break; + } + case GST_EVENT_GAP: + { + GstClockTime start, duration; + + gst_event_parse_gap (event, &start, &duration); + if (GST_CLOCK_TIME_IS_VALID (duration)) + start += duration; + /* we do not expect another buffer until after gap, + * so that is our position now */ + thiz->alpha_segment.position = start; + + /* wake up the video chain, it might be waiting for a alpha buffer or + * a alpha segment update */ + GST_ALPHA_MASK_LOCK (thiz); + GST_ALPHA_MASK_BROADCAST (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + + gst_event_unref (event); + ret = TRUE; + break; + } + case GST_EVENT_FLUSH_STOP: + GST_ALPHA_MASK_LOCK (thiz); + GST_INFO_OBJECT (thiz, "alpha flush stop"); + thiz->alpha_flushing = FALSE; + thiz->alpha_eos = FALSE; + gst_alpha_mask_pop_alpha (thiz); + gst_segment_init (&thiz->alpha_segment, GST_FORMAT_TIME); + GST_ALPHA_MASK_UNLOCK (thiz); + gst_event_unref (event); + ret = TRUE; + break; + case GST_EVENT_FLUSH_START: + GST_ALPHA_MASK_LOCK (thiz); + GST_INFO_OBJECT (thiz, "alpha flush start"); + thiz->alpha_flushing = TRUE; + GST_ALPHA_MASK_BROADCAST (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + gst_event_unref (event); + ret = TRUE; + break; + case GST_EVENT_EOS: + GST_ALPHA_MASK_LOCK (thiz); + thiz->alpha_eos = TRUE; + GST_INFO_OBJECT (thiz, "alpha EOS"); + /* wake up the video chain, it might be waiting for a alpha buffer or + * a alpha segment update */ + GST_ALPHA_MASK_BROADCAST (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + gst_event_unref (event); + ret = TRUE; + break; + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + + return ret; +} + +static GstPadLinkReturn +gst_alpha_mask_alpha_pad_link (GstPad * pad, GstObject * parent, GstPad * peer) +{ + GstAlphaMask *thiz; + + thiz = GST_ALPHA_MASK (parent); + if (G_UNLIKELY (!thiz)) + return GST_PAD_LINK_REFUSED; + + GST_DEBUG_OBJECT (thiz, "Alpha pad linked"); + + thiz->alpha_linked = TRUE; + + return GST_PAD_LINK_OK; +} + +static void +gst_alpha_mask_alpha_pad_unlink (GstPad * pad, GstObject * parent) +{ + GstAlphaMask *thiz; + + thiz = GST_ALPHA_MASK (parent); + + GST_DEBUG_OBJECT (thiz, "Alpha pad unlinked"); + + thiz->alpha_linked = FALSE; + + gst_segment_init (&thiz->alpha_segment, GST_FORMAT_UNDEFINED); +} + +static gboolean +gst_alpha_mask_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstAlphaMask *thiz; + gboolean ret; + + thiz = GST_ALPHA_MASK (parent); + + if (thiz->alpha_linked) { + gst_event_ref (event); + ret = gst_pad_push_event (thiz->video_sinkpad, event); + gst_pad_push_event (thiz->alpha_sinkpad, event); + } else { + ret = gst_pad_push_event (thiz->video_sinkpad, event); + } + + return ret; +} + +static GstStateChangeReturn +gst_alpha_mask_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + GstAlphaMask *thiz = GST_ALPHA_MASK (element); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + GST_ALPHA_MASK_LOCK (thiz); + thiz->alpha_flushing = TRUE; + thiz->video_flushing = TRUE; + /* pop_alpha will broadcast on the GCond and thus also make the video + * chain exit if it's waiting for a alpha buffer */ + gst_alpha_mask_pop_alpha (thiz); + GST_ALPHA_MASK_UNLOCK (thiz); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + GST_ALPHA_MASK_LOCK (thiz); + thiz->alpha_flushing = FALSE; + thiz->video_flushing = FALSE; + thiz->video_eos = FALSE; + thiz->alpha_eos = FALSE; + gst_segment_init (&thiz->segment, GST_FORMAT_TIME); + gst_segment_init (&thiz->alpha_segment, GST_FORMAT_TIME); + GST_ALPHA_MASK_UNLOCK (thiz); + break; + default: + break; + } + + return ret; +} + +static void +gst_alpha_mask_finalize (GObject * object) +{ + GstAlphaMask *thiz = GST_ALPHA_MASK (object); + + if (thiz->alpha_buffer) { + gst_buffer_unref (thiz->alpha_buffer); + thiz->alpha_buffer = NULL; + } + + if (thiz->convert) + gst_video_converter_free (thiz->convert); + thiz->convert = NULL; + + g_mutex_clear (&thiz->lock); + g_cond_clear (&thiz->cond); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_alpha_mask_class_init (GstAlphaMaskClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + + gobject_class->finalize = gst_alpha_mask_finalize; + + gst_element_class_set_static_metadata (gstelement_class, + "Alpha mask combinator", + "Filter/Effect/Video", + "Combines video and alpha streams", "Josep Torra <jtorra@oblong.com>"); + + gst_element_class_add_static_pad_template (gstelement_class, &vsink_factory); + gst_element_class_add_static_pad_template (gstelement_class, &asink_factory); + gst_element_class_add_static_pad_template (gstelement_class, &src_factory); + + gstelement_class->change_state = + GST_DEBUG_FUNCPTR (gst_alpha_mask_change_state); +} + +static void +gst_alpha_mask_init (GstAlphaMask * thiz) +{ + GstPadTemplate *template; + + /* video sink */ + template = gst_static_pad_template_get (&vsink_factory); + thiz->video_sinkpad = gst_pad_new_from_template (template, "video_sink"); + gst_object_unref (template); + gst_pad_set_chain_function (thiz->video_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_video_chain)); + gst_pad_set_event_function (thiz->video_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_video_event)); + gst_element_add_pad (GST_ELEMENT (thiz), thiz->video_sinkpad); + + /* alpha sink */ + template = gst_static_pad_template_get (&asink_factory); + thiz->alpha_sinkpad = gst_pad_new_from_template (template, "alpha_sink"); + gst_object_unref (template); + gst_pad_set_chain_function (thiz->alpha_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_alpha_chain)); + gst_pad_set_event_function (thiz->alpha_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_alpha_event)); + gst_pad_set_link_function (thiz->alpha_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_alpha_pad_link)); + gst_pad_set_unlink_function (thiz->alpha_sinkpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_alpha_pad_unlink)); + gst_element_add_pad (GST_ELEMENT (thiz), thiz->alpha_sinkpad); + + /* video source */ + template = gst_static_pad_template_get (&src_factory); + thiz->srcpad = gst_pad_new_from_template (template, "src"); + gst_object_unref (template); + gst_pad_set_event_function (thiz->srcpad, + GST_DEBUG_FUNCPTR (gst_alpha_mask_src_event)); + gst_element_add_pad (GST_ELEMENT (thiz), thiz->srcpad); + + thiz->convert = NULL; + thiz->alpha_buffer = NULL; + thiz->alpha_linked = FALSE; + + g_mutex_init (&thiz->lock); + g_cond_init (&thiz->cond); + gst_segment_init (&thiz->segment, GST_FORMAT_TIME); +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "alphamask", GST_RANK_NONE, + GST_TYPE_ALPHA_MASK); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + alphamask, + "Alpha mask combinator", + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/alpha/gstalphamask.h b/gst/alpha/gstalphamask.h new file mode 100644 index 000000000..60952bb3b --- /dev/null +++ b/gst/alpha/gstalphamask.h @@ -0,0 +1,90 @@ +/* GStreamer + * Copyright (C) 2016 Oblong Industries + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ALPHA_MASK_H__ +#define __GST_ALPHA_MASK_H__ + +#include <gst/gst.h> +#include <gst/video/video.h> + +G_BEGIN_DECLS + +#define GST_TYPE_ALPHA_MASK (gst_alpha_mask_get_type()) +#define GST_ALPHA_MASK(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),\ + GST_TYPE_ALPHA_MASK, GstAlphaMask)) +#define GST_ALPHA_MASK_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),\ + GST_TYPE_ALPHA_MASK, GstAlphaMaskClass)) +#define GST_ALPHA_MASK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj),\ + GST_TYPE_ALPHA_MASK, GstAlphaMaskClass)) +#define GST_IS_ALPHA_MASK(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),\ + GST_TYPE_ALPHA_MASK)) +#define GST_IS_ALPHA_MASK_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),\ + GST_TYPE_ALPHA_MASK)) + +typedef struct _GstAlphaMask GstAlphaMask; +typedef struct _GstAlphaMaskClass GstAlphaMaskClass; + +/** + * GstAlphaMask: + * + * Opaque alphamask object structure + */ +struct _GstAlphaMask { + GstElement element; + + GstPad *video_sinkpad; + GstPad *alpha_sinkpad; + GstPad *srcpad; + + GstSegment segment; + GstSegment alpha_segment; + GstBuffer *alpha_buffer; + gboolean alpha_linked; + gboolean video_flushing; + gboolean video_eos; + gboolean alpha_flushing; + gboolean alpha_eos; + + GMutex lock; + GCond cond; /* to signal removal of a queued alpha + * buffer, arrival of a alpha buffer, + * a alpha segment update, or a change + * in status (e.g. shutdown, flushing) */ + + /* stream details */ + GstVideoInfo iinfo; + GstVideoInfo ainfo; + GstVideoInfo oinfo; + gint width; + gint height; + GstVideoFormat iformat; + GstVideoFormat oformat; + + GstVideoConverter *convert; +}; + +struct _GstAlphaMaskClass { + GstElementClass parent_class; +}; + +GType gst_alpha_mask_get_type(void) G_GNUC_CONST; + +G_END_DECLS + +#endif /* __GST_ALPHA_MASK_H */ diff --git a/gst/alpha/meson.build b/gst/alpha/meson.build index 265525e46..0e5def4b0 100644 --- a/gst/alpha/meson.build +++ b/gst/alpha/meson.build @@ -13,3 +13,11 @@ gstalphacolor = library('gstalphacolor', 'gstalphacolor.c', install : true, install_dir : plugins_install_dir, ) + +gstalphamask = library('gstalphamask', 'gstalphamask.c', + c_args : gst_plugins_good_args, + include_directories : [configinc], + dependencies : [gstvideo_dep, gst_dep], + install : true, + install_dir : plugins_install_dir, +) |