Macintosh-SW/tenfourfox
1
0
Fork 0
mirror of https://github.com/classilla/tenfourfox.git synced 2024-06-15 22:29:43 +00:00
Code Issues Projects Releases Wiki Activity
f9fe116fa9
tenfourfox/dom/media/gstreamer/GStreamerReader.cpp
Cameron Kaiser c9b2922b70 hello FPR
2017-04-19 00:56:45 -07:00

1497 lines
48 KiB
C++
Raw Blame History

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsError.h"
#include "nsMimeTypes.h"
#include "MediaDecoderStateMachine.h"
#include "AbstractMediaDecoder.h"
#include "GStreamerReader.h"
#if GST_VERSION_MAJOR >= 1
#include "GStreamerAllocator.h"
#endif
#include "GStreamerFormatHelper.h"
#include "VideoUtils.h"
#include "mozilla/Endian.h"
#include "mozilla/Preferences.h"
#include "mozilla/unused.h"
#include "GStreamerLoader.h"
#include "gfx2DGlue.h"
namespace mozilla {
using namespace gfx;
using namespace layers;
using namespace media;
// Un-comment to enable logging of seek bisections.
//#define SEEK_LOGGING
extern LazyLogModule gMediaDecoderLog;
#define LOG(type, msg, ...) \
MOZ_LOG(gMediaDecoderLog, type, ("GStreamerReader(%p) " msg, this, ##__VA_ARGS__))
#if DEBUG
static const unsigned int MAX_CHANNELS = 4;
#endif
// Let the demuxer work in pull mode for short files. This used to be a micro
// optimization to have more accurate durations for ogg files in mochitests.
// Since as of today we aren't using gstreamer to demux ogg, and having demuxers
// work in pull mode over http makes them slower (since they really assume
// near-zero latency in pull mode) set the constant to 0 for now, which
// effectively disables it.
static const int SHORT_FILE_SIZE = 0;
// The default resource->Read() size when working in push mode
static const int DEFAULT_SOURCE_READ_SIZE = 50 * 1024;
typedef enum {
GST_PLAY_FLAG_VIDEO = (1 << 0),
GST_PLAY_FLAG_AUDIO = (1 << 1),
GST_PLAY_FLAG_TEXT = (1 << 2),
GST_PLAY_FLAG_VIS = (1 << 3),
GST_PLAY_FLAG_SOFT_VOLUME = (1 << 4),
GST_PLAY_FLAG_NATIVE_AUDIO = (1 << 5),
GST_PLAY_FLAG_NATIVE_VIDEO = (1 << 6),
GST_PLAY_FLAG_DOWNLOAD = (1 << 7),
GST_PLAY_FLAG_BUFFERING = (1 << 8),
GST_PLAY_FLAG_DEINTERLACE = (1 << 9),
GST_PLAY_FLAG_SOFT_COLORBALANCE = (1 << 10)
} PlayFlags;
GStreamerReader::GStreamerReader(AbstractMediaDecoder* aDecoder)
: MediaDecoderReader(aDecoder),
mMP3FrameParser(aDecoder->GetResource()->GetLength()),
mDataOffset(0),
mUseParserDuration(false),
mLastParserDuration(-1),
#if GST_VERSION_MAJOR >= 1
mAllocator(nullptr),
mBufferPool(nullptr),
#endif
mPlayBin(nullptr),
mBus(nullptr),
mSource(nullptr),
mVideoSink(nullptr),
mVideoAppSink(nullptr),
mAudioSink(nullptr),
mAudioAppSink(nullptr),
mFormat(GST_VIDEO_FORMAT_UNKNOWN),
mVideoSinkBufferCount(0),
mAudioSinkBufferCount(0),
mGstThreadsMonitor("media.gst.threads"),
mReachedAudioEos(false),
mReachedVideoEos(false),
#if GST_VERSION_MAJOR >= 1
mConfigureAlignment(true),
#endif
fpsNum(0),
fpsDen(0),
mResource(aDecoder->GetResource())
{
MOZ_COUNT_CTOR(GStreamerReader);
mSrcCallbacks.need_data = GStreamerReader::NeedDataCb;
mSrcCallbacks.enough_data = GStreamerReader::EnoughDataCb;
mSrcCallbacks.seek_data = GStreamerReader::SeekDataCb;
mSinkCallbacks.eos = GStreamerReader::EosCb;
mSinkCallbacks.new_preroll = GStreamerReader::NewPrerollCb;
#if GST_VERSION_MAJOR >= 1
mSinkCallbacks.new_sample = GStreamerReader::NewBufferCb;
#else
mSinkCallbacks.new_buffer = GStreamerReader::NewBufferCb;
mSinkCallbacks.new_buffer_list = nullptr;
#endif
gst_segment_init(&mVideoSegment, GST_FORMAT_UNDEFINED);
gst_segment_init(&mAudioSegment, GST_FORMAT_UNDEFINED);
}
GStreamerReader::~GStreamerReader()
{
MOZ_COUNT_DTOR(GStreamerReader);
NS_ASSERTION(!mPlayBin, "No Shutdown() after Init()");
}
nsresult GStreamerReader::InitInternal()
{
GStreamerFormatHelper::Instance();
#if GST_VERSION_MAJOR >= 1
mAllocator = static_cast<GstAllocator*>(g_object_new(GST_TYPE_MOZ_GFX_MEMORY_ALLOCATOR, nullptr));
moz_gfx_memory_allocator_set_reader(mAllocator, this);
mBufferPool = static_cast<GstBufferPool*>(g_object_new(GST_TYPE_MOZ_GFX_BUFFER_POOL, nullptr));
#endif
#if GST_VERSION_MAJOR >= 1
mPlayBin = gst_element_factory_make("playbin", nullptr);
#else
mPlayBin = gst_element_factory_make("playbin2", nullptr);
#endif
if (!mPlayBin) {
LOG(LogLevel::Error, "couldn't create playbin");
return NS_ERROR_FAILURE;
}
g_object_set(mPlayBin, "buffer-size", 0, nullptr);
mBus = gst_pipeline_get_bus(GST_PIPELINE(mPlayBin));
mVideoSink = gst_parse_bin_from_description("capsfilter name=filter ! "
"appsink name=videosink sync=false max-buffers=1 "
#if GST_VERSION_MAJOR >= 1
"caps=video/x-raw,format=I420"
#else
"caps=video/x-raw-yuv,format=(fourcc)I420"
#endif
, TRUE, nullptr);
mVideoAppSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(mVideoSink),
"videosink"));
mAudioSink = gst_parse_bin_from_description("capsfilter name=filter ! "
"appsink name=audiosink sync=false max-buffers=1", TRUE, nullptr);
mAudioAppSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(mAudioSink),
"audiosink"));
GstCaps* caps = BuildAudioSinkCaps();
g_object_set(mAudioAppSink, "caps", caps, nullptr);
gst_caps_unref(caps);
gst_app_sink_set_callbacks(mVideoAppSink, &mSinkCallbacks,
(gpointer) this, nullptr);
gst_app_sink_set_callbacks(mAudioAppSink, &mSinkCallbacks,
(gpointer) this, nullptr);
InstallPadCallbacks();
g_object_set(mPlayBin, "uri", "appsrc://",
"video-sink", mVideoSink,
"audio-sink", mAudioSink,
nullptr);
g_signal_connect(G_OBJECT(mPlayBin), "notify::source",
G_CALLBACK(GStreamerReader::PlayBinSourceSetupCb), this);
g_signal_connect(G_OBJECT(mPlayBin), "element-added",
G_CALLBACK(GStreamerReader::PlayElementAddedCb), this);
g_signal_connect(G_OBJECT(mPlayBin), "element-added",
G_CALLBACK(GStreamerReader::ElementAddedCb), this);
return NS_OK;
}
RefPtr<ShutdownPromise>
GStreamerReader::Shutdown()
{
ResetDecode();
if (mPlayBin) {
gst_app_src_end_of_stream(mSource);
if (mSource)
gst_object_unref(mSource);
gst_element_set_state(mPlayBin, GST_STATE_NULL);
gst_object_unref(mPlayBin);
mPlayBin = nullptr;
mVideoSink = nullptr;
mVideoAppSink = nullptr;
mAudioSink = nullptr;
mAudioAppSink = nullptr;
gst_object_unref(mBus);
mBus = nullptr;
#if GST_VERSION_MAJOR >= 1
g_object_unref(mAllocator);
g_object_unref(mBufferPool);
#endif
}
return MediaDecoderReader::Shutdown();
}
GstBusSyncReply
GStreamerReader::ErrorCb(GstBus *aBus, GstMessage *aMessage, gpointer aUserData)
{
return static_cast<GStreamerReader*>(aUserData)->Error(aBus, aMessage);
}
GstBusSyncReply
GStreamerReader::Error(GstBus *aBus, GstMessage *aMessage)
{
if (GST_MESSAGE_TYPE(aMessage) == GST_MESSAGE_ERROR) {
Eos();
}
return GST_BUS_PASS;
}
void GStreamerReader::ElementAddedCb(GstBin *aPlayBin,
GstElement *aElement,
gpointer aUserData)
{
GstElementFactory *factory = gst_element_get_factory(aElement);
if (!factory)
return;
const gchar *name = gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(factory));
if (name && !strcmp(name, "uridecodebin")) {
g_signal_connect(G_OBJECT(aElement), "autoplug-sort",
G_CALLBACK(GStreamerReader::ElementFilterCb), aUserData);
}
}
GValueArray *GStreamerReader::ElementFilterCb(GstURIDecodeBin *aBin,
GstPad *aPad,
GstCaps *aCaps,
GValueArray *aFactories,
gpointer aUserData)
{
return ((GStreamerReader*)aUserData)->ElementFilter(aBin, aPad, aCaps, aFactories);
}
GValueArray *GStreamerReader::ElementFilter(GstURIDecodeBin *aBin,
GstPad *aPad,
GstCaps *aCaps,
GValueArray *aFactories)
{
GValueArray *filtered = g_value_array_new(aFactories->n_values);
for (unsigned int i = 0; i < aFactories->n_values; i++) {
GValue *value = &aFactories->values[i];
GstPluginFeature *factory = GST_PLUGIN_FEATURE(g_value_peek_pointer(value));
if (!GStreamerFormatHelper::IsPluginFeatureBlocked(factory)) {
g_value_array_append(filtered, value);
}
}
return filtered;
}
void GStreamerReader::PlayBinSourceSetupCb(GstElement* aPlayBin,
GParamSpec* pspec,
gpointer aUserData)
{
GstElement *source;
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
g_object_get(aPlayBin, "source", &source, nullptr);
reader->PlayBinSourceSetup(GST_APP_SRC(source));
}
void GStreamerReader::PlayBinSourceSetup(GstAppSrc* aSource)
{
mSource = GST_APP_SRC(aSource);
gst_app_src_set_callbacks(mSource, &mSrcCallbacks, (gpointer) this, nullptr);
/* do a short read to trigger a network request so that GetLength() below
* returns something meaningful and not -1
*/
char buf[512];
unsigned int size = 0;
mResource.Read(buf, sizeof(buf), &size);
mResource.Seek(SEEK_SET, 0);
/* now we should have a length */
int64_t resourceLength = GetDataLength();
gst_app_src_set_size(mSource, resourceLength);
if (mResource.GetResource()->IsDataCachedToEndOfResource(0) ||
(resourceLength != -1 && resourceLength <= SHORT_FILE_SIZE)) {
/* let the demuxer work in pull mode for local files (or very short files)
* so that we get optimal seeking accuracy/performance
*/
LOG(LogLevel::Debug, "configuring random access, len %lld", resourceLength);
gst_app_src_set_stream_type(mSource, GST_APP_STREAM_TYPE_RANDOM_ACCESS);
} else {
/* make the demuxer work in push mode so that seeking is kept to a minimum
*/
LOG(LogLevel::Debug, "configuring push mode, len %lld", resourceLength);
gst_app_src_set_stream_type(mSource, GST_APP_STREAM_TYPE_SEEKABLE);
}
// Set the source MIME type to stop typefind trying every. single. format.
GstCaps *caps =
GStreamerFormatHelper::ConvertFormatsToCaps(mDecoder->GetResource()->GetContentType().get(),
nullptr);
gst_app_src_set_caps(aSource, caps);
gst_caps_unref(caps);
}
/**
* If this stream is an MP3, we want to parse the headers to estimate the
* stream duration.
*/
nsresult GStreamerReader::ParseMP3Headers()
{
const uint32_t MAX_READ_BYTES = 4096;
uint64_t offset = 0;
char bytes[MAX_READ_BYTES];
uint32_t bytesRead;
do {
nsresult rv = mResource.ReadAt(offset, bytes, MAX_READ_BYTES, &bytesRead);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(bytesRead, NS_ERROR_FAILURE);
mMP3FrameParser.Parse(reinterpret_cast<uint8_t*>(bytes), bytesRead, offset);
offset += bytesRead;
} while (!mMP3FrameParser.ParsedHeaders());
if (mMP3FrameParser.IsMP3()) {
mLastParserDuration = mMP3FrameParser.GetDuration();
mDataOffset = mMP3FrameParser.GetMP3Offset();
// Update GStreamer's stream length in case we found any ID3 headers to
// ignore.
gst_app_src_set_size(mSource, GetDataLength());
}
return NS_OK;
}
int64_t
GStreamerReader::GetDataLength()
{
int64_t streamLen = mResource.GetLength();
if (streamLen < 0) {
return streamLen;
}
return streamLen - mDataOffset;
}
nsresult GStreamerReader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
MOZ_ASSERT(OnTaskQueue());
nsresult ret = NS_OK;
/*
* Parse MP3 headers before we kick off the GStreamer pipeline otherwise there
* might be concurrent stream operations happening on both decoding and gstreamer
* threads which will screw the GStreamer state machine.
*/
LOG(LogLevel::Debug, "content-type: %s %s",
mDecoder->GetResource()->GetContentType().get(),
mDecoder->GetResource()->GetContentURL().get());
bool isMP3 = mDecoder->GetResource()->GetContentType().EqualsASCII(AUDIO_MP3);
if (isMP3) {
ParseMP3Headers();
}
/* We do 3 attempts here: decoding audio and video, decoding video only,
* decoding audio only. This allows us to play streams that have one broken
* stream but that are otherwise decodeable.
*/
guint flags[3] = {GST_PLAY_FLAG_VIDEO|GST_PLAY_FLAG_AUDIO,
static_cast<guint>(~GST_PLAY_FLAG_AUDIO), static_cast<guint>(~GST_PLAY_FLAG_VIDEO)};
guint default_flags, current_flags;
g_object_get(mPlayBin, "flags", &default_flags, nullptr);
GstMessage* message = nullptr;
for (unsigned int i = 0; i < G_N_ELEMENTS(flags); i++) {
current_flags = default_flags & flags[i];
g_object_set(G_OBJECT(mPlayBin), "flags", current_flags, nullptr);
/* reset filter caps to ANY */
GstCaps* caps = gst_caps_new_any();
GstElement* filter = gst_bin_get_by_name(GST_BIN(mAudioSink), "filter");
g_object_set(filter, "caps", caps, nullptr);
gst_object_unref(filter);
filter = gst_bin_get_by_name(GST_BIN(mVideoSink), "filter");
g_object_set(filter, "caps", caps, nullptr);
gst_object_unref(filter);
gst_caps_unref(caps);
filter = nullptr;
if (!(current_flags & GST_PLAY_FLAG_AUDIO))
filter = gst_bin_get_by_name(GST_BIN(mAudioSink), "filter");
else if (!(current_flags & GST_PLAY_FLAG_VIDEO))
filter = gst_bin_get_by_name(GST_BIN(mVideoSink), "filter");
if (filter) {
/* Little trick: set the target caps to "skip" so that playbin2 fails to
* find a decoder for the stream we want to skip.
*/
GstCaps* filterCaps = gst_caps_new_simple ("skip", nullptr, nullptr);
g_object_set(filter, "caps", filterCaps, nullptr);
gst_caps_unref(filterCaps);
gst_object_unref(filter);
}
LOG(LogLevel::Debug, "starting metadata pipeline");
if (gst_element_set_state(mPlayBin, GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) {
LOG(LogLevel::Debug, "metadata pipeline state change failed");
ret = NS_ERROR_FAILURE;
continue;
}
/* Wait for ASYNC_DONE, which is emitted when the pipeline is built,
* prerolled and ready to play. Also watch for errors.
*/
message = gst_bus_timed_pop_filtered(mBus, GST_CLOCK_TIME_NONE,
(GstMessageType)(GST_MESSAGE_ASYNC_DONE | GST_MESSAGE_ERROR | GST_MESSAGE_EOS));
if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ASYNC_DONE) {
LOG(LogLevel::Debug, "read metadata pipeline prerolled");
gst_message_unref(message);
ret = NS_OK;
break;
} else {
LOG(LogLevel::Debug, "read metadata pipeline failed to preroll: %s",
gst_message_type_get_name (GST_MESSAGE_TYPE (message)));
if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) {
GError* error;
gchar* debug;
gst_message_parse_error(message, &error, &debug);
LOG(LogLevel::Error, "read metadata error: %s: %s", error->message, debug);
g_error_free(error);
g_free(debug);
}
/* Unexpected stream close/EOS or other error. We'll give up if all
* streams are in error/eos. */
gst_element_set_state(mPlayBin, GST_STATE_NULL);
gst_message_unref(message);
ret = NS_ERROR_FAILURE;
}
}
if (NS_SUCCEEDED(ret))
ret = CheckSupportedFormats();
if (NS_FAILED(ret))
/* we couldn't get this to play */
return ret;
/* report the duration */
gint64 duration;
bool isMediaSeekable = false;
if (isMP3 && mMP3FrameParser.IsMP3()) {
// The MP3FrameParser has reported a duration; use that over the gstreamer
// reported duration for inter-platform consistency.
mUseParserDuration = true;
mLastParserDuration = mMP3FrameParser.GetDuration();
mInfo.mMetadataDuration.emplace(TimeUnit::FromMicroseconds(mLastParserDuration));
isMediaSeekable = true;
} else {
LOG(LogLevel::Debug, "querying duration");
// Otherwise use the gstreamer duration.
#if GST_VERSION_MAJOR >= 1
if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
GST_FORMAT_TIME, &duration)) {
#else
GstFormat format = GST_FORMAT_TIME;
if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
&format, &duration) && format == GST_FORMAT_TIME) {
#endif
LOG(LogLevel::Debug, "have duration %" GST_TIME_FORMAT, GST_TIME_ARGS(duration));
duration = GST_TIME_AS_USECONDS (duration);
mInfo.mMetadataDuration.emplace(TimeUnit::FromMicroseconds(duration));
isMediaSeekable = true;
}
}
mInfo.mMediaSeekable = isMediaSeekable;
int n_video = 0, n_audio = 0;
g_object_get(mPlayBin, "n-video", &n_video, "n-audio", &n_audio, nullptr);
if (!n_video) {
mInfo.mVideo = VideoInfo();
}
if (!n_audio) {
mInfo.mAudio = AudioInfo();
}
*aInfo = mInfo;
*aTags = nullptr;
// Watch the pipeline for fatal errors
#if GST_VERSION_MAJOR >= 1
gst_bus_set_sync_handler(mBus, GStreamerReader::ErrorCb, this, nullptr);
#else
gst_bus_set_sync_handler(mBus, GStreamerReader::ErrorCb, this);
#endif
/* set the pipeline to PLAYING so that it starts decoding and queueing data in
* the appsinks */
gst_element_set_state(mPlayBin, GST_STATE_PLAYING);
return NS_OK;
}
nsresult GStreamerReader::CheckSupportedFormats()
{
bool done = false;
bool unsupported = false;
GstIterator* it = gst_bin_iterate_recurse(GST_BIN(mPlayBin));
while (!done) {
GstIteratorResult res;
GstElement* element;
#if GST_VERSION_MAJOR >= 1
GValue value = {0,};
res = gst_iterator_next(it, &value);
#else
res = gst_iterator_next(it, (void **) &element);
#endif
switch(res) {
case GST_ITERATOR_OK:
{
#if GST_VERSION_MAJOR >= 1
element = GST_ELEMENT (g_value_get_object (&value));
#endif
GstElementFactory* factory = gst_element_get_factory(element);
if (factory) {
const char* klass = gst_element_factory_get_klass(factory);
GstPad* pad = gst_element_get_static_pad(element, "sink");
if (pad) {
GstCaps* caps;
#if GST_VERSION_MAJOR >= 1
caps = gst_pad_get_current_caps(pad);
#else
caps = gst_pad_get_negotiated_caps(pad);
#endif
if (caps) {
/* check for demuxers but ignore elements like id3demux */
if (strstr (klass, "Demuxer") && !strstr(klass, "Metadata"))
unsupported = !GStreamerFormatHelper::Instance()->CanHandleContainerCaps(caps);
else if (strstr (klass, "Decoder") && !strstr(klass, "Generic"))
unsupported = !GStreamerFormatHelper::Instance()->CanHandleCodecCaps(caps);
gst_caps_unref(caps);
}
gst_object_unref(pad);
}
}
#if GST_VERSION_MAJOR >= 1
g_value_unset (&value);
#else
gst_object_unref(element);
#endif
done = unsupported;
break;
}
case GST_ITERATOR_RESYNC:
unsupported = false;
break;
case GST_ITERATOR_ERROR:
done = true;
break;
case GST_ITERATOR_DONE:
done = true;
break;
}
}
gst_iterator_free(it);
return unsupported ? NS_ERROR_FAILURE : NS_OK;
}
nsresult GStreamerReader::ResetDecode()
{
nsresult res = NS_OK;
LOG(LogLevel::Debug, "reset decode");
if (NS_FAILED(MediaDecoderReader::ResetDecode())) {
res = NS_ERROR_FAILURE;
}
mVideoQueue.Reset();
mAudioQueue.Reset();
mVideoSinkBufferCount = 0;
mAudioSinkBufferCount = 0;
mReachedAudioEos = false;
mReachedVideoEos = false;
#if GST_VERSION_MAJOR >= 1
mConfigureAlignment = true;
#endif
LOG(LogLevel::Debug, "reset decode done");
return res;
}
bool GStreamerReader::DecodeAudioData()
{
MOZ_ASSERT(OnTaskQueue());
GstBuffer *buffer = nullptr;
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
if (mReachedAudioEos && !mAudioSinkBufferCount) {
return false;
}
/* Wait something to be decoded before return or continue */
if (!mAudioSinkBufferCount) {
if(!mVideoSinkBufferCount) {
/* We have nothing decoded so it makes no sense to return to the state machine
* as it will call us back immediately, we'll return again and so on, wasting
* CPU cycles for no job done. So, block here until there is either video or
* audio data available
*/
mon.Wait();
if (!mAudioSinkBufferCount) {
/* There is still no audio data available, so either there is video data or
* something else has happened (Eos, etc...). Return to the state machine
* to process it.
*/
return true;
}
}
else {
return true;
}
}
#if GST_VERSION_MAJOR >= 1
GstSample *sample = gst_app_sink_pull_sample(mAudioAppSink);
buffer = gst_buffer_ref(gst_sample_get_buffer(sample));
gst_sample_unref(sample);
#else
buffer = gst_app_sink_pull_buffer(mAudioAppSink);
#endif
mAudioSinkBufferCount--;
}
int64_t timestamp = GST_BUFFER_TIMESTAMP(buffer);
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
timestamp = gst_segment_to_stream_time(&mAudioSegment,
GST_FORMAT_TIME, timestamp);
}
timestamp = GST_TIME_AS_USECONDS(timestamp);
int64_t offset = GST_BUFFER_OFFSET(buffer);
guint8* data;
#if GST_VERSION_MAJOR >= 1
GstMapInfo info;
gst_buffer_map(buffer, &info, GST_MAP_READ);
unsigned int size = info.size;
data = info.data;
#else
unsigned int size = GST_BUFFER_SIZE(buffer);
data = GST_BUFFER_DATA(buffer);
#endif
int32_t frames = (size / sizeof(AudioDataValue)) / mInfo.mAudio.mChannels;
typedef AudioCompactor::NativeCopy GstCopy;
mAudioCompactor.Push(offset,
timestamp,
mInfo.mAudio.mRate,
frames,
mInfo.mAudio.mChannels,
GstCopy(data,
size,
mInfo.mAudio.mChannels));
#if GST_VERSION_MAJOR >= 1
gst_buffer_unmap(buffer, &info);
#endif
gst_buffer_unref(buffer);
return true;
}
bool GStreamerReader::DecodeVideoFrame(bool &aKeyFrameSkip,
int64_t aTimeThreshold)
{
MOZ_ASSERT(OnTaskQueue());
GstBuffer *buffer = nullptr;
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
if (mReachedVideoEos && !mVideoSinkBufferCount) {
return false;
}
/* Wait something to be decoded before return or continue */
if (!mVideoSinkBufferCount) {
if (!mAudioSinkBufferCount) {
/* We have nothing decoded so it makes no sense to return to the state machine
* as it will call us back immediately, we'll return again and so on, wasting
* CPU cycles for no job done. So, block here until there is either video or
* audio data available
*/
mon.Wait();
if (!mVideoSinkBufferCount) {
/* There is still no video data available, so either there is audio data or
* something else has happened (Eos, etc...). Return to the state machine
* to process it
*/
return true;
}
}
else {
return true;
}
}
mDecoder->NotifyDecodedFrames(0, 1, 0);
#if GST_VERSION_MAJOR >= 1
GstSample *sample = gst_app_sink_pull_sample(mVideoAppSink);
buffer = gst_buffer_ref(gst_sample_get_buffer(sample));
gst_sample_unref(sample);
#else
buffer = gst_app_sink_pull_buffer(mVideoAppSink);
#endif
mVideoSinkBufferCount--;
}
bool isKeyframe = !GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
if ((aKeyFrameSkip && !isKeyframe)) {
mDecoder->NotifyDecodedFrames(0, 0, 1);
gst_buffer_unref(buffer);
return true;
}
int64_t timestamp = GST_BUFFER_TIMESTAMP(buffer);
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
timestamp = gst_segment_to_stream_time(&mVideoSegment,
GST_FORMAT_TIME, timestamp);
}
NS_ASSERTION(GST_CLOCK_TIME_IS_VALID(timestamp),
"frame has invalid timestamp");
timestamp = GST_TIME_AS_USECONDS(timestamp);
int64_t duration = 0;
if (GST_CLOCK_TIME_IS_VALID(GST_BUFFER_DURATION(buffer)))
duration = GST_TIME_AS_USECONDS(GST_BUFFER_DURATION(buffer));
else if (fpsNum && fpsDen)
/* add 1-frame duration */
duration = gst_util_uint64_scale(GST_USECOND, fpsDen, fpsNum);
if (timestamp < aTimeThreshold) {
LOG(LogLevel::Debug, "skipping frame %" GST_TIME_FORMAT
" threshold %" GST_TIME_FORMAT,
GST_TIME_ARGS(timestamp * 1000),
GST_TIME_ARGS(aTimeThreshold * 1000));
gst_buffer_unref(buffer);
return true;
}
if (!buffer)
/* no more frames */
return true;
#if GST_VERSION_MAJOR >= 1
if (mConfigureAlignment && buffer->pool) {
GstStructure *config = gst_buffer_pool_get_config(buffer->pool);
GstVideoAlignment align;
if (gst_buffer_pool_config_get_video_alignment(config, &align))
gst_video_info_align(&mVideoInfo, &align);
gst_structure_free(config);
mConfigureAlignment = false;
}
#endif
RefPtr<PlanarYCbCrImage> image = GetImageFromBuffer(buffer);
if (!image) {
/* Ugh, upstream is not calling gst_pad_alloc_buffer(). Fallback to
* allocating a PlanarYCbCrImage backed GstBuffer here and memcpy.
*/
GstBuffer* tmp = nullptr;
CopyIntoImageBuffer(buffer, &tmp, image);
gst_buffer_unref(buffer);
buffer = tmp;
}
int64_t offset = mResource.Tell(); // Estimate location in media.
RefPtr<VideoData> video = VideoData::CreateFromImage(mInfo.mVideo,
mDecoder->GetImageContainer(),
offset, timestamp, duration,
static_cast<Image*>(image.get()),
isKeyframe, -1, mPicture);
mVideoQueue.Push(video);
gst_buffer_unref(buffer);
return true;
}
RefPtr<MediaDecoderReader::SeekPromise>
GStreamerReader::Seek(int64_t aTarget, int64_t aEndTime)
{
MOZ_ASSERT(OnTaskQueue());
gint64 seekPos = aTarget * GST_USECOND;
LOG(LogLevel::Debug, "%p About to seek to %" GST_TIME_FORMAT,
mDecoder, GST_TIME_ARGS(seekPos));
int flags = GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT;
if (!gst_element_seek_simple(mPlayBin,
GST_FORMAT_TIME,
static_cast<GstSeekFlags>(flags),
seekPos)) {
LOG(LogLevel::Error, "seek failed");
return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
}
LOG(LogLevel::Debug, "seek succeeded");
GstMessage* message = gst_bus_timed_pop_filtered(mBus, GST_CLOCK_TIME_NONE,
(GstMessageType)(GST_MESSAGE_ASYNC_DONE | GST_MESSAGE_ERROR));
gst_message_unref(message);
LOG(LogLevel::Debug, "seek completed");
return SeekPromise::CreateAndResolve(aTarget, __func__);
}
media::TimeIntervals GStreamerReader::GetBuffered()
{
MOZ_ASSERT(OnTaskQueue());
if (!HaveStartTime()) {
return media::TimeIntervals();
}
media::TimeIntervals buffered;
if (!mInfo.HasValidMedia()) {
return buffered;
}
#if GST_VERSION_MAJOR == 0
GstFormat format = GST_FORMAT_TIME;
#endif
AutoPinned<MediaResource> resource(mDecoder->GetResource());
MediaByteRangeSet ranges;
resource->GetCachedRanges(ranges);
if (resource->IsDataCachedToEndOfResource(0)) {
/* fast path for local or completely cached files */
gint64 duration =
mDuration.Ref().refOr(media::TimeUnit::FromMicroseconds(0)).ToMicroseconds();
LOG(LogLevel::Debug, "complete range [0, %f] for [0, %li]",
(double) duration / GST_MSECOND, GetDataLength());
buffered +=
media::TimeInterval(media::TimeUnit::FromMicroseconds(0),
media::TimeUnit::FromMicroseconds(duration));
return buffered;
}
for(uint32_t index = 0; index < ranges.Length(); index++) {
int64_t startOffset = ranges[index].mStart;
int64_t endOffset = ranges[index].mEnd;
gint64 startTime, endTime, duration;
bool haveDuration = false;
#if GST_VERSION_MAJOR >= 1
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
startOffset, GST_FORMAT_TIME, &startTime))
continue;
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
endOffset, GST_FORMAT_TIME, &endTime))
continue;
if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
GST_FORMAT_TIME, &duration)) {
haveDuration = true;
}
#else
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
startOffset, &format, &startTime) || format != GST_FORMAT_TIME)
continue;
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
endOffset, &format, &endTime) || format != GST_FORMAT_TIME)
continue;
if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
&format, &duration) && format == GST_FORMAT_TIME) {
haveDuration = true;
}
#endif
// Check that the estimated time doesn't go beyond known duration
// as this indicates a buggy gst plugin.
if (haveDuration && endTime > duration) {
LOG(LogLevel::Debug, "Have duration %" GST_TIME_FORMAT "contradicting endTime %" GST_TIME_FORMAT,
GST_TIME_ARGS(duration), GST_TIME_ARGS(endTime));
endTime = std::min(endTime, duration);
}
LOG(LogLevel::Debug, "adding range [%f, %f] for [%li %li] size %li",
(double) GST_TIME_AS_USECONDS (startTime) / GST_MSECOND,
(double) GST_TIME_AS_USECONDS (endTime) / GST_MSECOND,
startOffset, endOffset, GetDataLength());
buffered +=
media::TimeInterval(media::TimeUnit::FromMicroseconds(GST_TIME_AS_USECONDS(startTime)),
media::TimeUnit::FromMicroseconds(GST_TIME_AS_USECONDS(endTime)));
}
return buffered;
}
void GStreamerReader::ReadAndPushData(guint aLength)
{
int64_t offset1 = mResource.Tell();
Unused << offset1;
nsresult rv = NS_OK;
GstBuffer* buffer = gst_buffer_new_and_alloc(aLength);
#if GST_VERSION_MAJOR >= 1
GstMapInfo info;
gst_buffer_map(buffer, &info, GST_MAP_WRITE);
guint8 *data = info.data;
#else
guint8* data = GST_BUFFER_DATA(buffer);
#endif
uint32_t size = 0, bytesRead = 0;
while(bytesRead < aLength) {
rv = mResource.Read(reinterpret_cast<char*>(data + bytesRead),
aLength - bytesRead, &size);
if (NS_FAILED(rv) || size == 0)
break;
bytesRead += size;
}
int64_t offset2 = mResource.Tell();
Unused << offset2;
#if GST_VERSION_MAJOR >= 1
gst_buffer_unmap(buffer, &info);
gst_buffer_set_size(buffer, bytesRead);
#else
GST_BUFFER_SIZE(buffer) = bytesRead;
#endif
GstFlowReturn ret = gst_app_src_push_buffer(mSource, gst_buffer_ref(buffer));
if (ret != GST_FLOW_OK) {
LOG(LogLevel::Error, "ReadAndPushData push ret %s(%d)", gst_flow_get_name(ret), ret);
}
if (NS_FAILED(rv)) {
/* Terminate the stream if there is an error in reading */
LOG(LogLevel::Error, "ReadAndPushData read error, rv=%x", rv);
gst_app_src_end_of_stream(mSource);
} else if (bytesRead < aLength) {
/* If we read less than what we wanted, we reached the end */
LOG(LogLevel::Warning, "ReadAndPushData read underflow, "
"bytesRead=%u, aLength=%u, offset(%lld,%lld)",
bytesRead, aLength, offset1, offset2);
gst_app_src_end_of_stream(mSource);
}
gst_buffer_unref(buffer);
/* Ensure offset change is consistent in this function.
* If there are other stream operations on another thread at the same time,
* it will disturb the GStreamer state machine.
*/
MOZ_ASSERT(offset1 + bytesRead == offset2);
}
void GStreamerReader::NeedDataCb(GstAppSrc* aSrc,
guint aLength,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->NeedData(aSrc, aLength);
}
void GStreamerReader::NeedData(GstAppSrc* aSrc, guint aLength)
{
if (aLength == static_cast<guint>(-1))
aLength = DEFAULT_SOURCE_READ_SIZE;
ReadAndPushData(aLength);
}
void GStreamerReader::EnoughDataCb(GstAppSrc* aSrc, gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->EnoughData(aSrc);
}
void GStreamerReader::EnoughData(GstAppSrc* aSrc)
{
}
gboolean GStreamerReader::SeekDataCb(GstAppSrc* aSrc,
guint64 aOffset,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
return reader->SeekData(aSrc, aOffset);
}
gboolean GStreamerReader::SeekData(GstAppSrc* aSrc, guint64 aOffset)
{
aOffset += mDataOffset;
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
int64_t resourceLength = mResource.GetLength();
if (gst_app_src_get_size(mSource) == -1) {
/* It's possible that we didn't know the length when we initialized mSource
* but maybe we do now
*/
gst_app_src_set_size(mSource, GetDataLength());
}
nsresult rv = NS_ERROR_FAILURE;
if (aOffset < static_cast<guint64>(resourceLength)) {
rv = mResource.Seek(SEEK_SET, aOffset);
}
return NS_SUCCEEDED(rv);
}
GstFlowReturn GStreamerReader::NewPrerollCb(GstAppSink* aSink,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
if (aSink == reader->mVideoAppSink)
reader->VideoPreroll();
else
reader->AudioPreroll();
return GST_FLOW_OK;
}
void GStreamerReader::AudioPreroll()
{
/* The first audio buffer has reached the audio sink. Get rate and channels */
LOG(LogLevel::Debug, "Audio preroll");
GstPad* sinkpad = gst_element_get_static_pad(GST_ELEMENT(mAudioAppSink), "sink");
#if GST_VERSION_MAJOR >= 1
GstCaps *caps = gst_pad_get_current_caps(sinkpad);
#else
GstCaps* caps = gst_pad_get_negotiated_caps(sinkpad);
#endif
GstStructure* s = gst_caps_get_structure(caps, 0);
mInfo.mAudio.mRate = mInfo.mAudio.mChannels = 0;
gst_structure_get_int(s, "rate", (gint*) &mInfo.mAudio.mRate);
gst_structure_get_int(s, "channels", (gint*) &mInfo.mAudio.mChannels);
NS_ASSERTION(mInfo.mAudio.mRate != 0, ("audio rate is zero"));
NS_ASSERTION(mInfo.mAudio.mChannels != 0, ("audio channels is zero"));
NS_ASSERTION(mInfo.mAudio.mChannels > 0 && mInfo.mAudio.mChannels <= MAX_CHANNELS,
"invalid audio channels number");
gst_caps_unref(caps);
gst_object_unref(sinkpad);
}
void GStreamerReader::VideoPreroll()
{
/* The first video buffer has reached the video sink. Get width and height */
LOG(LogLevel::Debug, "Video preroll");
GstPad* sinkpad = gst_element_get_static_pad(GST_ELEMENT(mVideoAppSink), "sink");
int PARNumerator, PARDenominator;
#if GST_VERSION_MAJOR >= 1
GstCaps* caps = gst_pad_get_current_caps(sinkpad);
memset (&mVideoInfo, 0, sizeof (mVideoInfo));
gst_video_info_from_caps(&mVideoInfo, caps);
mFormat = mVideoInfo.finfo->format;
mPicture.width = mVideoInfo.width;
mPicture.height = mVideoInfo.height;
PARNumerator = GST_VIDEO_INFO_PAR_N(&mVideoInfo);
PARDenominator = GST_VIDEO_INFO_PAR_D(&mVideoInfo);
#else
GstCaps* caps = gst_pad_get_negotiated_caps(sinkpad);
gst_video_format_parse_caps(caps, &mFormat, &mPicture.width, &mPicture.height);
if (!gst_video_parse_caps_pixel_aspect_ratio(caps, &PARNumerator, &PARDenominator)) {
PARNumerator = 1;
PARDenominator = 1;
}
#endif
NS_ASSERTION(mPicture.width && mPicture.height, "invalid video resolution");
// Calculate display size according to pixel aspect ratio.
nsIntRect pictureRect(0, 0, mPicture.width, mPicture.height);
nsIntSize frameSize = nsIntSize(mPicture.width, mPicture.height);
nsIntSize displaySize = nsIntSize(mPicture.width, mPicture.height);
ScaleDisplayByAspectRatio(displaySize, float(PARNumerator) / float(PARDenominator));
// If video frame size is overflow, stop playing.
if (IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
GstStructure* structure = gst_caps_get_structure(caps, 0);
gst_structure_get_fraction(structure, "framerate", &fpsNum, &fpsDen);
mInfo.mVideo.mDisplay = displaySize;
} else {
LOG(LogLevel::Debug, "invalid video region");
Eos();
}
gst_caps_unref(caps);
gst_object_unref(sinkpad);
}
GstFlowReturn GStreamerReader::NewBufferCb(GstAppSink* aSink,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
if (aSink == reader->mVideoAppSink)
reader->NewVideoBuffer();
else
reader->NewAudioBuffer();
return GST_FLOW_OK;
}
void GStreamerReader::NewVideoBuffer()
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* We have a new video buffer queued in the video sink. Increment the counter
* and notify the decode thread potentially blocked in DecodeVideoFrame
*/
mDecoder->NotifyDecodedFrames(1, 0, 0);
mVideoSinkBufferCount++;
mon.NotifyAll();
}
void GStreamerReader::NewAudioBuffer()
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* We have a new audio buffer queued in the audio sink. Increment the counter
* and notify the decode thread potentially blocked in DecodeAudioData
*/
mAudioSinkBufferCount++;
mon.NotifyAll();
}
void GStreamerReader::EosCb(GstAppSink* aSink, gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->Eos(aSink);
}
void GStreamerReader::Eos(GstAppSink* aSink)
{
/* We reached the end of the stream */
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* Potentially unblock DecodeVideoFrame and DecodeAudioData */
if (aSink == mVideoAppSink) {
mReachedVideoEos = true;
} else if (aSink == mAudioAppSink) {
mReachedAudioEos = true;
} else {
// Assume this is an error causing an EOS.
mReachedAudioEos = true;
mReachedVideoEos = true;
}
mon.NotifyAll();
}
}
/**
* This callback is called while the pipeline is automatically built, after a
* new element has been added to the pipeline. We use it to find the
* uridecodebin instance used by playbin and connect to it to apply our
* block list.
*/
void
GStreamerReader::PlayElementAddedCb(GstBin *aBin, GstElement *aElement,
gpointer *aUserData)
{
const static char sUriDecodeBinPrefix[] = "uridecodebin";
gchar *name = gst_element_get_name(aElement);
// Attach this callback to uridecodebin, child of playbin.
if (!strncmp(name, sUriDecodeBinPrefix, sizeof(sUriDecodeBinPrefix) - 1)) {
g_signal_connect(G_OBJECT(aElement), "autoplug-sort",
G_CALLBACK(GStreamerReader::AutoplugSortCb), aUserData);
}
g_free(name);
}
bool
GStreamerReader::ShouldAutoplugFactory(GstElementFactory* aFactory, GstCaps* aCaps)
{
bool autoplug;
const gchar *klass = gst_element_factory_get_klass(aFactory);
if (strstr(klass, "Demuxer") && !strstr(klass, "Metadata")) {
autoplug = GStreamerFormatHelper::Instance()->CanHandleContainerCaps(aCaps);
} else if (strstr(klass, "Decoder") && !strstr(klass, "Generic")) {
autoplug = GStreamerFormatHelper::Instance()->CanHandleCodecCaps(aCaps);
} else {
/* we only filter demuxers and decoders, let everything else be autoplugged */
autoplug = true;
}
return autoplug;
}
/**
* This is called by uridecodebin (running inside playbin), after it has found
* candidate factories to continue decoding the stream. We apply the block list
* here, disallowing known-crashy plugins.
*/
GValueArray*
GStreamerReader::AutoplugSortCb(GstElement* aElement, GstPad* aPad,
GstCaps* aCaps, GValueArray* aFactories)
{
if (!aFactories->n_values) {
return nullptr;
}
/* aFactories[0] is the element factory that is going to be used to
* create the next element needed to demux or decode the stream.
*/
GstElementFactory *factory = (GstElementFactory*) g_value_get_object(g_value_array_get_nth(aFactories, 0));
if (!ShouldAutoplugFactory(factory, aCaps)) {
/* We don't support this factory. Return an empty array to signal that we
* don't want to continue decoding this (sub)stream.
*/
return g_value_array_new(0);
}
/* nullptr means that we're ok with the candidates and don't need to apply any
* sorting/filtering.
*/
return nullptr;
}
/**
* If this is an MP3 stream, pass any new data we get to the MP3 frame parser
* for duration estimation.
*/
void GStreamerReader::NotifyDataArrivedInternal()
{
MOZ_ASSERT(OnTaskQueue());
if (HasVideo()) {
return;
}
if (!mMP3FrameParser.NeedsData()) {
return;
}
AutoPinned<MediaResource> resource(mResource.GetResource());
MediaByteRangeSet byteRanges;
nsresult rv = resource->GetCachedRanges(byteRanges);
if (NS_FAILED(rv)) {
return;
}
if (byteRanges == mLastCachedRanges) {
return;
}
MediaByteRangeSet intervals = byteRanges - mLastCachedRanges;
mLastCachedRanges = byteRanges;
for (const auto& interval : intervals) {
RefPtr<MediaByteBuffer> bytes =
resource->MediaReadAt(interval.mStart, interval.Length());
NS_ENSURE_TRUE_VOID(bytes);
mMP3FrameParser.Parse(bytes->Elements(), interval.Length(), interval.mStart);
if (!mMP3FrameParser.IsMP3()) {
return;
}
}
int64_t duration = mMP3FrameParser.GetDuration();
if (duration != mLastParserDuration && mUseParserDuration) {
MOZ_ASSERT(mDecoder);
mLastParserDuration = duration;
mDecoder->DispatchUpdateEstimatedMediaDuration(mLastParserDuration);
}
}
#if GST_VERSION_MAJOR >= 1
GstCaps* GStreamerReader::BuildAudioSinkCaps()
{
GstCaps* caps = gst_caps_from_string("audio/x-raw, channels={1,2}");
const char* format;
#ifdef MOZ_SAMPLE_TYPE_FLOAT32
#if MOZ_LITTLE_ENDIAN
format = "F32LE";
#else
format = "F32BE";
#endif
#else /* !MOZ_SAMPLE_TYPE_FLOAT32 */
#if MOZ_LITTLE_ENDIAN
format = "S16LE";
#else
format = "S16BE";
#endif
#endif
gst_caps_set_simple(caps, "format", G_TYPE_STRING, format, nullptr);
return caps;
}
void GStreamerReader::InstallPadCallbacks()
{
GstPad* sinkpad = gst_element_get_static_pad(GST_ELEMENT(mVideoAppSink), "sink");
gst_pad_add_probe(sinkpad,
(GstPadProbeType) (GST_PAD_PROBE_TYPE_SCHEDULING |
GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM |
GST_PAD_PROBE_TYPE_EVENT_UPSTREAM |
GST_PAD_PROBE_TYPE_EVENT_FLUSH),
&GStreamerReader::EventProbeCb, this, nullptr);
gst_pad_add_probe(sinkpad, GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM,
GStreamerReader::QueryProbeCb, nullptr, nullptr);
gst_pad_set_element_private(sinkpad, this);
gst_object_unref(sinkpad);
sinkpad = gst_element_get_static_pad(GST_ELEMENT(mAudioAppSink), "sink");
gst_pad_add_probe(sinkpad,
(GstPadProbeType) (GST_PAD_PROBE_TYPE_SCHEDULING |
GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM |
GST_PAD_PROBE_TYPE_EVENT_UPSTREAM |
GST_PAD_PROBE_TYPE_EVENT_FLUSH),
&GStreamerReader::EventProbeCb, this, nullptr);
gst_object_unref(sinkpad);
}
GstPadProbeReturn GStreamerReader::EventProbeCb(GstPad *aPad,
GstPadProbeInfo *aInfo,
gpointer aUserData)
{
GStreamerReader *reader = (GStreamerReader *) aUserData;
GstEvent *aEvent = (GstEvent *)aInfo->data;
return reader->EventProbe(aPad, aEvent);
}
GstPadProbeReturn GStreamerReader::EventProbe(GstPad *aPad, GstEvent *aEvent)
{
GstElement* parent = GST_ELEMENT(gst_pad_get_parent(aPad));
LOG(LogLevel::Debug, "event probe %s", GST_EVENT_TYPE_NAME (aEvent));
switch(GST_EVENT_TYPE(aEvent)) {
case GST_EVENT_SEGMENT:
{
const GstSegment *newSegment;
GstSegment* segment;
/* Store the segments so we can convert timestamps to stream time, which
* is what the upper layers sync on.
*/
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
#if GST_VERSION_MINOR <= 1 && GST_VERSION_MICRO < 1
ResetDecode();
#endif
gst_event_parse_segment(aEvent, &newSegment);
if (parent == GST_ELEMENT(mVideoAppSink))
segment = &mVideoSegment;
else
segment = &mAudioSegment;
gst_segment_copy_into (newSegment, segment);
break;
}
case GST_EVENT_FLUSH_STOP:
/* Reset on seeks */
ResetDecode();
break;
default:
break;
}
gst_object_unref(parent);
return GST_PAD_PROBE_OK;
}
GstPadProbeReturn GStreamerReader::QueryProbeCb(GstPad* aPad, GstPadProbeInfo* aInfo, gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(gst_pad_get_element_private(aPad));
return reader->QueryProbe(aPad, aInfo, aUserData);
}
GstPadProbeReturn GStreamerReader::QueryProbe(GstPad* aPad, GstPadProbeInfo* aInfo, gpointer aUserData)
{
GstQuery *query = gst_pad_probe_info_get_query(aInfo);
GstPadProbeReturn ret = GST_PAD_PROBE_OK;
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_ALLOCATION:
GstCaps *caps;
GstVideoInfo info;
gboolean need_pool;
gst_query_parse_allocation(query, &caps, &need_pool);
gst_video_info_init(&info);
gst_video_info_from_caps(&info, caps);
gst_query_add_allocation_param(query, mAllocator, nullptr);
gst_query_add_allocation_pool(query, mBufferPool, info.size, 0, 0);
break;
default:
break;
}
return ret;
}
void GStreamerReader::ImageDataFromVideoFrame(GstVideoFrame *aFrame,
PlanarYCbCrImage::Data *aData)
{
NS_ASSERTION(GST_VIDEO_INFO_IS_YUV(&mVideoInfo),
"Non-YUV video frame formats not supported");
NS_ASSERTION(GST_VIDEO_FRAME_N_COMPONENTS(aFrame) == 3,
"Unsupported number of components in video frame");
aData->mPicX = aData->mPicY = 0;
aData->mPicSize = gfx::IntSize(mPicture.width, mPicture.height);
aData->mStereoMode = StereoMode::MONO;
aData->mYChannel = GST_VIDEO_FRAME_COMP_DATA(aFrame, 0);
aData->mYStride = GST_VIDEO_FRAME_COMP_STRIDE(aFrame, 0);
aData->mYSize = gfx::IntSize(GST_VIDEO_FRAME_COMP_WIDTH(aFrame, 0),
GST_VIDEO_FRAME_COMP_HEIGHT(aFrame, 0));
aData->mYSkip = GST_VIDEO_FRAME_COMP_PSTRIDE(aFrame, 0) - 1;
aData->mCbCrStride = GST_VIDEO_FRAME_COMP_STRIDE(aFrame, 1);
aData->mCbCrSize = gfx::IntSize(GST_VIDEO_FRAME_COMP_WIDTH(aFrame, 1),
GST_VIDEO_FRAME_COMP_HEIGHT(aFrame, 1));
aData->mCbChannel = GST_VIDEO_FRAME_COMP_DATA(aFrame, 1);
aData->mCrChannel = GST_VIDEO_FRAME_COMP_DATA(aFrame, 2);
aData->mCbSkip = GST_VIDEO_FRAME_COMP_PSTRIDE(aFrame, 1) - 1;
aData->mCrSkip = GST_VIDEO_FRAME_COMP_PSTRIDE(aFrame, 2) - 1;
}
RefPtr<PlanarYCbCrImage> GStreamerReader::GetImageFromBuffer(GstBuffer* aBuffer)
{
RefPtr<PlanarYCbCrImage> image = nullptr;
if (gst_buffer_n_memory(aBuffer) == 1) {
GstMemory* mem = gst_buffer_peek_memory(aBuffer, 0);
if (GST_IS_MOZ_GFX_MEMORY_ALLOCATOR(mem->allocator)) {
image = moz_gfx_memory_get_image(mem);
GstVideoFrame frame;
gst_video_frame_map(&frame, &mVideoInfo, aBuffer, GST_MAP_READ);
PlanarYCbCrImage::Data data;
ImageDataFromVideoFrame(&frame, &data);
image->SetDataNoCopy(data);
gst_video_frame_unmap(&frame);
}
}
return image;
}
void GStreamerReader::CopyIntoImageBuffer(GstBuffer* aBuffer,
GstBuffer** aOutBuffer,
RefPtr<PlanarYCbCrImage> &image)
{
*aOutBuffer = gst_buffer_new_allocate(mAllocator, gst_buffer_get_size(aBuffer), nullptr);
GstMemory *mem = gst_buffer_peek_memory(*aOutBuffer, 0);
GstMapInfo map_info;
gst_memory_map(mem, &map_info, GST_MAP_WRITE);
gst_buffer_extract(aBuffer, 0, map_info.data, gst_buffer_get_size(aBuffer));
gst_memory_unmap(mem, &map_info);
/* create a new gst buffer with the newly created memory and copy the
* metadata over from the incoming buffer */
gst_buffer_copy_into(*aOutBuffer, aBuffer,
(GstBufferCopyFlags)(GST_BUFFER_COPY_METADATA), 0, -1);
image = GetImageFromBuffer(*aOutBuffer);
}
#endif
} // namespace mozilla
Reference in New Issue View Git Blame Copy Permalink