(61d00a474) v0.9.7.1

Libraries/webm_mem_playback/webm_mem_playback/AudioDecoder.cpp

@@ -0,0 +1,15 @@
+#include "AudioDecoder.h"
+
+OpusAudioDecoder::OpusAudioDecoder(int smpRate, int channels) {
+	int errorCode = OPUS_OK;
+	sampleRate = smpRate; channelCount = channels;
+	decoder = opus_decoder_create(sampleRate, channelCount, &errorCode);
+}
+
+OpusAudioDecoder::~OpusAudioDecoder() {
+	opus_decoder_destroy(decoder);
+}
+
+int OpusAudioDecoder::decode(uint8_t* compressedBuf, int compressedSize, int16_t* outBuf, int outSize) {
+	return opus_decode(decoder, compressedBuf, compressedSize, outBuf, outSize / channelCount, 0) * channelCount;
+}

Libraries/webm_mem_playback/webm_mem_playback/AudioDecoder.h

@@ -0,0 +1,29 @@
+#ifndef AUDIODECODER_H_INCLUDED
+#define AUDIODECODER_H_INCLUDED
+
+//opus
+#include <opus.h>
+
+//stl
+#include <cinttypes>
+
+class AudioDecoder {
+	public:
+		virtual ~AudioDecoder() {}
+		virtual int decode(uint8_t* compressedBuf, int compressedSize, int16_t* outBuf, int outSize) =0;
+};
+
+class OpusAudioDecoder : public AudioDecoder {
+	public:
+		OpusAudioDecoder(int smpRate, int channels);
+		virtual ~OpusAudioDecoder();
+		int decode(uint8_t* compressedBuf, int compressedSize, int16_t* outBuf, int outSize) override;
+	private:
+		OpusDecoder* decoder;
+		int channelCount;
+		int sampleRate;
+};
+
+/* TODO: implement vorbis decoder */
+
+#endif

Libraries/webm_mem_playback/webm_mem_playback/Exports.cpp

@@ -0,0 +1,54 @@
+#include "Exports.h"
+#include "Video.h"
+
+Video* loadVideo(const char* filename) {
+	Video* retVal = new Video(filename);
+	if (retVal->getErrorCode() != 0) {
+		delete retVal; retVal = nullptr;
+	}
+	return retVal;
+}
+
+int32_t getVideoWidth(Video* video) {
+	return video->getWidth();
+}
+
+int32_t getVideoHeight(Video* video) {
+	return video->getHeight();
+}
+
+int32_t videoHasAudio(Video* video) {
+	return video->hasAudio() ? 1 : 0;
+}
+
+int32_t getVideoAudioSampleRate(Video* video) {
+	return video->getAudioSampleRate();
+}
+
+int32_t getVideoAudioChannelCount(Video* video) {
+	return video->getAudioChannelCount();
+}
+
+void deleteVideo(Video* video) {
+	delete video;
+}
+
+void playVideo(Video* video) {
+	video->play();
+}
+
+void stopVideo(Video* video) {
+	video->stop();
+}
+
+int32_t isVideoPlaying(Video* video) {
+	return video->isPlaying() ? 1 : 0;
+}
+
+void setVideoFrameCallback(Video* video, EventCallback callback) {
+	video->setVideoCallback(callback);
+}
+
+void setVideoAudioCallback(Video* video, EventCallback callback) {
+	video->setAudioCallback(callback);
+}

Libraries/webm_mem_playback/webm_mem_playback/Exports.h

@@ -0,0 +1,30 @@
+#ifndef EXPORTS_H_INCLUDED
+#define EXPORTS_H_INCLUDED
+
+#ifdef _WIN32
+#define EXPORT(x) extern "C" _declspec(dllexport) x  _cdecl
+#elif defined __APPLE__
+#define EXPORT(x) extern "C" x  __attribute__((visibility("default")))
+#else
+#define EXPORT(x) extern "C" x
+#endif
+
+#include <cinttypes>
+
+class Video;
+typedef void (*EventCallback)(Video* video, void* data, int32_t dataElemSize, int32_t dataLen);
+
+EXPORT(Video*) loadVideo(const char* filename);
+EXPORT(int32_t) getVideoWidth(Video* video);
+EXPORT(int32_t) getVideoHeight(Video* video);
+EXPORT(int32_t) videoHasAudio(Video* video);
+EXPORT(int32_t) getVideoAudioSampleRate(Video* video);
+EXPORT(int32_t) getVideoAudioChannelCount(Video* video);
+EXPORT(void) deleteVideo(Video* video);
+EXPORT(void) playVideo(Video* video);
+EXPORT(void) stopVideo(Video* video);
+EXPORT(int32_t) isVideoPlaying(Video* video);
+EXPORT(void) setVideoFrameCallback(Video* video, EventCallback callback);
+EXPORT(void) setVideoAudioCallback(Video* video, EventCallback callback);
+
+#endif

Libraries/webm_mem_playback/webm_mem_playback/Makefile

@@ -0,0 +1,15 @@
+CXX=g++
+CXXFLAGS= -O3 -fPIC -Wall -I../libvpx_x64_linux -I../libwebm_x64_linux -I../opus/include
+
+
+webm_mem_playback_x64: Exports.o AudioDecoder.o Video.o
+	$(CXX) $(CXXFLAGS) -shared -L../opus_x64_linux/.libs/ -L../libvpx_x64_linux/ -L../libwebm_x64_linux/ -Wl,--whole-archive -lopus -lvpx -lwebm -lpthread -Wl,--no-whole-archive -o webm_mem_playback_x64.so Exports.o AudioDecoder.o Video.o
+
+Exports.o: Exports.cpp Exports.h Video.h
+	$(CXX) $(CXXFLAGS) -c Exports.cpp
+
+Video.o: Video.cpp AudioDecoder.h Video.h
+	$(CXX) $(CXXFLAGS) -c Video.cpp
+
+AudioDecoder.o: AudioDecoder.cpp AudioDecoder.h
+	$(CXX) $(CXXFLAGS) -c AudioDecoder.cpp

Libraries/webm_mem_playback/webm_mem_playback/Video.cpp

@@ -0,0 +1,359 @@
+#include "Video.h"
+
+static const VpxInterface vpx_decoders[] = {
+#if CONFIG_VP8_DECODER
+  { "vp8", VP8_FOURCC, &vpx_codec_vp8_dx },
+#endif
+
+#if CONFIG_VP9_DECODER
+  { "vp9", VP9_FOURCC, &vpx_codec_vp9_dx },
+#endif
+};
+
+int get_vpx_decoder_count(void) {
+	return sizeof(vpx_decoders) / sizeof(vpx_decoders[0]);
+}
+
+//https://en.wikipedia.org/wiki/YUV#Y%E2%80%B2UV444_to_RGB888_conversion
+static void frameYUV420toRGBA(vpx_image_t* img, uint32_t* buf) {
+	for (int y = 0; y < img->d_h; y++) {
+		for (int x = 0; x < img->d_w; x++) {
+			int c = img->planes[0][(y * img->stride[0]) + x] - 16;
+			int d = img->planes[1][((y/2) * img->stride[1]) + (x/2)] - 128;
+			int e = img->planes[2][((y/2) * img->stride[2]) + (x/2)] - 128;
+			
+			int r = (298 * c + 409 * e + 128) >> 8;
+			if (r > 255) { r = 255; } if (r < 0) { r = 0; }
+			int g = (298 * c - 100 * d - 208 * e + 128) >> 8;
+			if (g > 255) { g = 255; } if (g < 0) { g = 0; }
+			int b = (298 * c + 516 * d + 128) >> 8;
+			if (b > 255) { b = 255; } if (b < 0) { b = 0; }
+
+			buf[x + y * img->d_w] = (255 << 24) | (b << 16) | (g << 8) | r;
+		}
+	}
+}
+
+Video::Video(const char* fn) {
+	mkvReader = nullptr;
+	parserSegment = nullptr;
+	parserSegmentInfo = nullptr;
+
+	filename = fn;
+	
+	videoPlaybackPos = 0; audioPlaybackPos = 0;
+	videoBlockIndex = 0; audioBlockIndex = 0;
+
+	mkvReader = new mkvparser::MkvReader();
+
+	errorCode = mkvReader->Open(fn);
+	if (errorCode) { return; }
+
+	errorCode = ebmlHeader.Parse(mkvReader, ebmlHeaderPos);
+	if (errorCode) { return; }
+
+	errorCode = mkvparser::Segment::CreateInstance(mkvReader, ebmlHeaderPos, parserSegment);
+	if (errorCode) { return; }
+
+	errorCode = parserSegment->Load();
+	if (errorCode < 0) { return; }
+
+	parserSegmentInfo = parserSegment->GetInfo();
+	if (parserSegmentInfo == nullptr) { errorCode = -1; return; }
+
+	const mkvparser::Tracks* const parserTracks = parserSegment->GetTracks();
+	
+	mkvparser::VideoTrack* videoTrack = nullptr;
+	mkvparser::AudioTrack* audioTrack = nullptr;
+	unsigned long trackCount = parserTracks->GetTracksCount();
+	for (int i = 0; i < trackCount; i++) {
+		const mkvparser::Track* track = parserTracks->GetTrackByIndex(i);
+
+		if (track->GetType() == mkvparser::Track::Type::kVideo) {
+			if (videoTrack == nullptr) { videoTrack = (mkvparser::VideoTrack*)track; }
+		} else if (track->GetType() == mkvparser::Track::Type::kAudio) {
+			if (audioTrack == nullptr) { audioTrack = (mkvparser::AudioTrack*)track; }
+		}
+	}
+
+	if (videoTrack == nullptr) { errorCode = -1; return; }
+	
+	//framerate = videoTrack->GetFrameRate(); //TODO: reimplement?
+	width = videoTrack->GetDisplayWidth();
+	height = videoTrack->GetDisplayHeight();
+	videoCodecName = videoTrack->GetCodecId();
+
+	rawFrameData = new uint32_t[width*height];
+
+	int videoCodecIndex = -1;
+	if (videoCodecName == mkvmuxer::Tracks::kVp8CodecId) {
+		videoCodecIndex = 0;
+	} else if (videoCodecName == mkvmuxer::Tracks::kVp9CodecId) {
+		videoCodecIndex = 1;
+	}
+
+	vpxCodecConfig.w = width;
+	vpxCodecConfig.h = height;
+	vpxCodecConfig.threads = 1;
+
+	vpx_codec_dec_init(&vpxCodec, vpx_decoders[videoCodecIndex].codec_interface(), &vpxCodecConfig, 0);
+
+	audioAvail = audioTrack != nullptr;
+
+	audioSampleRate = 0; audioChannelCount = 0;
+	audioDecoder = nullptr;
+	rawAudioDataSize = 0;
+	rawAudioData = nullptr;
+	audioCodecName = "N/A";
+	if (audioAvail) {
+		audioCodecName = audioTrack->GetCodecId();
+		if (audioCodecName == mkvmuxer::Tracks::kOpusCodecId) {
+			audioDecoder = new OpusAudioDecoder((int)audioTrack->GetSamplingRate(), audioTrack->GetChannels());
+		} else {
+			audioCodecName = "Unsupported Codec "+audioCodecName;
+			audioAvail = false;
+		}
+	}
+
+	if (audioAvail) {
+		rawAudioDataSize = audioTrack->GetSamplingRate()*audioTrack->GetChannels() * 2;
+		rawAudioData = new int16_t[rawAudioDataSize];
+		audioSampleRate = audioTrack->GetSamplingRate();
+		audioChannelCount = audioTrack->GetChannels();
+	}
+
+	long long prevVideoTime = 0;
+	long long prevAudioTime = 0;
+	const mkvparser::Cluster* cluster = parserSegment->GetFirst();
+
+	while (cluster != nullptr) {
+		const mkvparser::BlockEntry* blockEntry;
+		cluster->GetFirst(blockEntry);
+
+		for (; blockEntry != nullptr; cluster->GetNext(blockEntry, blockEntry)) {
+			const mkvparser::Block* block = blockEntry->GetBlock();
+			mkvparser::Track::Type blockType = (mkvparser::Track::Type)parserTracks->GetTrackByNumber(block->GetTrackNumber())->GetType();
+			
+			if (block->GetFrameCount() <= 0) { continue; }
+			if (blockType == mkvparser::Track::Type::kVideo) {
+				videoBlocks.push_back(block);
+				videoBlockTimes.push_back(prevVideoTime);
+				prevVideoTime = block->GetTime(cluster);
+			} else if (blockType == mkvparser::Track::Type::kAudio && audioAvail) {
+				audioBlocks.push_back(block);
+				audioBlockTimes.push_back(prevAudioTime);
+				prevAudioTime = block->GetTime(cluster);
+			}
+
+			if (blockEntry->EOS()) { break; }
+		}
+
+		if (cluster->EOS()) { break; }
+		cluster = parserSegment->GetNext(cluster);
+	}
+
+	compressedFrameData = nullptr;
+	compressedFrameCapacity = 0;
+	compressedAudioData = nullptr;
+	compressedAudioCapacity = 0;
+	stopRequested = true;
+	videoThread = nullptr;
+	audioThread = nullptr;
+	stop();
+}
+
+static void initUpdateVideoThread(Video* video) {
+	video->updateVideo();
+}
+
+static void initUpdateAudioThread(Video* video) {
+	video->updateAudio();
+}
+
+void Video::updateVideo() {
+	long long makeupTime = 0;
+	for (; videoBlockIndex < videoBlocks.size(); videoBlockIndex++) {
+		if (stopRequested) { break; }
+
+		int i = videoBlockIndex;
+		const mkvparser::Block* block = videoBlocks[i];
+		for (int j = 0; j < block->GetFrameCount(); j++) {
+            std::chrono::high_resolution_clock::time_point frameTargetEndTime = std::chrono::high_resolution_clock::now();
+			long long frameDuration = 0;
+			if (videoBlockIndex < videoBlocks.size() - 1) {
+				//TODO: does it even make sense for a block to have more than one frame?
+				frameDuration = (videoBlockTimes[i + 1] - videoBlockTimes[i]) / block->GetFrameCount();
+			}
+			frameTargetEndTime += std::chrono::nanoseconds(frameDuration-1);
+			const mkvparser::Block::Frame& blockFrame = block->GetFrame(j);
+
+			long long frameDataPos = blockFrame.pos;
+			long long frameDataLen = blockFrame.len;
+
+			if (frameDataLen <= 0) { break; }
+
+			if (compressedFrameData == nullptr) {
+				compressedFrameData = new uint8_t[frameDataLen*2];
+				compressedFrameCapacity = frameDataLen * 2;
+			} else if (compressedFrameCapacity < frameDataLen) {
+				delete[] compressedFrameData;
+				compressedFrameData = new uint8_t[frameDataLen*2];
+				compressedFrameCapacity = frameDataLen * 2;
+			}
+
+			mkvReaderMutex.lock();
+			mkvReader->Read(frameDataPos, frameDataLen, compressedFrameData);
+			mkvReaderMutex.unlock();
+
+			vpx_codec_decode(&vpxCodec, compressedFrameData, frameDataLen, this, 0);
+
+			if (makeupTime>0) {
+				makeupTime -= frameDuration;
+				videoPlaybackPos += frameDuration;
+				continue;
+			}
+
+			vpx_codec_iter_t iter = NULL;
+			vpx_image_t* img = NULL;
+			img = vpx_codec_get_frame(&vpxCodec, &iter);
+			if (img == nullptr) { continue; }
+
+			frameYUV420toRGBA(img, rawFrameData);
+
+			if (videoCallback != nullptr) { videoCallback(this, rawFrameData, sizeof(uint32_t), width*height); }
+
+			std::chrono::high_resolution_clock::time_point frameEndTime = std::chrono::high_resolution_clock::now();
+			if ((frameEndTime - frameTargetEndTime).count() > 0) {
+				makeupTime += (frameEndTime - frameTargetEndTime).count() * 2;
+			} else {
+				std::this_thread::sleep_until(frameTargetEndTime);
+			}
+			videoPlaybackPos += frameDuration;
+		}
+	}
+	stopRequested = true;
+}
+
+void Video::updateAudio() {
+	for (; audioBlockIndex < audioBlocks.size(); audioBlockIndex++) {
+		if (stopRequested) { break; }
+
+		int i = audioBlockIndex;
+		const mkvparser::Block* block = audioBlocks[i];
+		for (int j = 0; j < block->GetFrameCount(); j++) {
+			long long frameDuration = 0;
+			if (audioBlockIndex < audioBlocks.size() - 1) {
+				//TODO: does it even make sense for a block to have more than one frame?
+				frameDuration = (audioBlockTimes[i + 1] - audioBlockTimes[i]) / block->GetFrameCount();
+			}
+			std::chrono::high_resolution_clock::time_point frameTargetEndTime = std::chrono::high_resolution_clock::now() + std::chrono::nanoseconds(frameDuration-1000000);
+			const mkvparser::Block::Frame& blockFrame = block->GetFrame(j);
+
+			long long frameDataPos = blockFrame.pos;
+			long long frameDataLen = blockFrame.len;
+
+			if (frameDataLen <= 0) { break; }
+
+			if (compressedAudioData == nullptr) {
+				compressedAudioData = new uint8_t[frameDataLen * 2];
+				compressedAudioCapacity = frameDataLen * 2;
+			}
+			else if (compressedAudioCapacity < frameDataLen) {
+				delete[] compressedAudioData;
+				compressedAudioData = new uint8_t[frameDataLen * 2];
+				compressedAudioCapacity = frameDataLen * 2;
+			}
+
+			mkvReaderMutex.lock();
+			mkvReader->Read(frameDataPos, frameDataLen, compressedAudioData);
+			mkvReaderMutex.unlock();
+
+			int decodedSampleCount = audioDecoder->decode(compressedAudioData, frameDataLen, rawAudioData, rawAudioDataSize);
+
+			if (audioCallback != nullptr) { audioCallback(this, rawAudioData, sizeof(int16_t), decodedSampleCount); }
+
+			std::chrono::high_resolution_clock::time_point frameEndTime = std::chrono::high_resolution_clock::now();
+			if ((frameEndTime - frameTargetEndTime).count() < 0) {
+                //disabled this because timing issues in the audio are much more noticeable than the video
+				//std::this_thread::sleep_until(frameTargetEndTime);
+			}
+			audioPlaybackPos += frameDuration;
+		}
+	}
+}
+
+Video::~Video() {
+	stop();
+	if (audioDecoder != nullptr) { delete audioDecoder; }
+	vpx_codec_destroy(&vpxCodec);
+	if (parserSegment != nullptr) {
+		delete parserSegment;
+	}
+	if (mkvReader != nullptr) {
+		mkvReader->Close(); delete mkvReader; mkvReader = nullptr;
+	}
+	if (rawFrameData != nullptr) {
+		delete[] rawFrameData;
+	}
+	if (rawAudioData != nullptr) {
+		delete[] rawAudioData;
+	}
+    if (compressedFrameData != nullptr) {
+        delete[] compressedFrameData;
+    }
+    if (compressedAudioData != nullptr) {
+        delete[] compressedAudioData;
+    }
+}
+
+void Video::play() {
+	stop(); resume();
+}
+
+void Video::stop() {
+	pause();
+	videoPlaybackPos = 0;
+	audioPlaybackPos = 0;
+	videoBlockIndex = 0;
+	audioBlockIndex = 0;
+}
+
+void Video::pause() {
+	stopRequested = true;
+	if (videoThread != nullptr) {
+		videoThread->join();
+		delete videoThread;
+		videoThread = nullptr;
+	}
+	if (audioThread != nullptr) {
+		audioThread->join();
+		delete audioThread;
+		audioThread = nullptr;
+	}
+}
+
+void Video::resume() {
+	//TODO: perform video-audio synchronization here
+	stopRequested = false;
+	videoThread = new std::thread(initUpdateVideoThread, this);
+	if (audioAvail) { audioThread = new std::thread(initUpdateAudioThread, this); }
+}
+
+long Video::getPlaybackPos() { return videoPlaybackPos; }
+void Video::seek(long pos) { /* TODO: implement? */ }
+
+int Video::getWidth() { return width; }
+int Video::getHeight() { return height; }
+//double Video::getFramerate() { return framerate; }
+bool Video::hasAudio() { return audioAvail; }
+int Video::getAudioSampleRate() { return audioSampleRate; }
+int Video::getAudioChannelCount() { return audioChannelCount; }
+
+const char* Video::getVideoCodec() { return videoCodecName.c_str(); }
+
+bool Video::isPlaying() { return !stopRequested; }
+
+int Video::getErrorCode() { return errorCode; }
+
+void Video::setVideoCallback(EventCallback callback) { videoCallback = callback; }
+void Video::setAudioCallback(EventCallback callback) { audioCallback = callback; }

Libraries/webm_mem_playback/webm_mem_playback/Video.h

@@ -0,0 +1,110 @@
+#ifndef VIDEO_H_INCLUDED
+#define VIDEO_H_INCLUDED
+
+//std
+#include <thread>
+#include <mutex>
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+
+//libvpx, libwebm
+#include <common/file_util.h>
+#include <common/hdr_util.h>
+#include <mkvmuxer/mkvmuxer.h>
+#include <mkvparser/mkvparser.h>
+#include <mkvparser/mkvreader.h>
+#include <vpx/vpx_decoder.h>
+#include <tools_common.h>
+#include <vpx/vp8dx.h>
+
+//opus
+#include <opus.h>
+
+//local headers
+#include "AudioDecoder.h"
+#include "Exports.h"
+
+class Video {
+	public:
+		Video(const char* fn);
+		~Video();
+
+		void play();
+		void stop();
+		void resume();
+		void pause();
+		long getPlaybackPos();
+		void seek(long pos);
+
+		int getWidth();
+		int getHeight();
+		//double getFramerate();
+		bool hasAudio();
+		int getAudioSampleRate();
+		int getAudioChannelCount();
+		const char* getVideoCodec();
+
+		bool isPlaying();
+
+		int getErrorCode();
+
+		void setVideoCallback(EventCallback callback);
+		void setAudioCallback(EventCallback callback);
+
+		void updateVideo();
+		void updateAudio();
+	private:
+		mkvparser::MkvReader* mkvReader;
+		std::mutex mkvReaderMutex;
+		long long ebmlHeaderPos;  mkvparser::EBMLHeader ebmlHeader;
+		mkvparser::Segment* parserSegment;
+		const mkvparser::SegmentInfo* parserSegmentInfo;
+
+		std::vector<const mkvparser::Block*> videoBlocks;
+		std::vector<long long> videoBlockTimes;
+		std::vector<const mkvparser::Block*> audioBlocks;
+		std::vector<long long> audioBlockTimes;
+
+		int width; int height; /*double framerate;*/
+		
+		bool audioAvail; int audioSampleRate; int audioChannelCount;
+
+		std::string filename;
+		std::string videoCodecName;
+		std::string audioCodecName;
+
+		vpx_codec_ctx_t vpxCodec;
+		vpx_codec_dec_cfg_t vpxCodecConfig;
+
+		EventCallback videoCallback = nullptr;
+		EventCallback audioCallback = nullptr;
+
+		uint8_t* compressedFrameData;
+		int compressedFrameCapacity;
+
+		uint8_t* compressedAudioData;
+		int compressedAudioCapacity;
+
+		uint32_t* rawFrameData;
+
+		int rawAudioDataSize;
+		int16_t* rawAudioData;
+
+		std::thread* videoThread;
+		std::thread* audioThread;
+
+		std::atomic<long long> videoPlaybackPos;
+		std::atomic<long long> audioPlaybackPos;
+		std::atomic<int> videoBlockIndex;
+		std::atomic<int> audioBlockIndex;
+		std::atomic<bool> stopRequested;
+
+		AudioDecoder* audioDecoder;
+
+		long long errorCode;
+};
+
+
+#endif // !VIDEO_H_INCLUDED