CoreAudio::HLE: Add FFmpeg aac decoder

This commit is contained in:
B3N30 2018-12-09 22:25:45 +01:00
parent bf1dbb47dc
commit c521f3b7d6
14 changed files with 850 additions and 5 deletions

View file

@ -5,6 +5,8 @@ add_library(audio_core STATIC
dsp_interface.cpp
dsp_interface.h
hle/common.h
hle/decoder.cpp
hle/decoder.h
hle/filter.cpp
hle/filter.h
hle/hle.cpp
@ -27,6 +29,7 @@ add_library(audio_core STATIC
$<$<BOOL:${SDL2_FOUND}>:sdl2_sink.cpp sdl2_sink.h>
$<$<BOOL:${ENABLE_CUBEB}>:cubeb_sink.cpp cubeb_sink.h>
$<$<BOOL:${FFMPEG_FOUND}>:hle/aac_decoder.cpp hle/aac_decoder.h hle/ffmpeg_dl.h>
)
create_target_directory_groups(audio_core)
@ -43,3 +46,13 @@ if(ENABLE_CUBEB)
target_link_libraries(audio_core PRIVATE cubeb)
add_definitions(-DHAVE_CUBEB=1)
endif()
if(FFMPEG_FOUND)
if(UNIX)
target_link_libraries(audio_core PRIVATE FFmpeg::avcodec)
else()
target_include_directories(audio_core PRIVATE ${FFMPEG_DIR}/include)
endif()
target_compile_definitions(audio_core PRIVATE HAVE_FFMPEG)
endif()

View file

@ -0,0 +1,241 @@
// Copyright 2018 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include "audio_core/hle/aac_decoder.h"
#include "audio_core/hle/ffmpeg_dl.h"
namespace AudioCore::HLE {
class AACDecoder::Impl {
public:
Impl(Memory::MemorySystem& memory);
~Impl();
std::optional<BinaryResponse> ProcessRequest(const BinaryRequest& request);
private:
std::optional<BinaryResponse> Initalize(const BinaryRequest& request);
void Clear();
std::optional<BinaryResponse> Decode(const BinaryRequest& request);
bool initalized;
bool have_ffmpeg_dl;
Memory::MemorySystem& memory;
AVCodec* codec;
AVCodecContext* av_context = nullptr;
AVCodecParserContext* parser = nullptr;
AVPacket* av_packet;
AVFrame* decoded_frame = nullptr;
};
AACDecoder::Impl::Impl(Memory::MemorySystem& memory) : memory(memory) {
initalized = false;
have_ffmpeg_dl = InitFFmpegDL();
}
AACDecoder::Impl::~Impl() {
if (initalized)
Clear();
}
std::optional<BinaryResponse> AACDecoder::Impl::ProcessRequest(const BinaryRequest& request) {
if (request.codec != DecoderCodec::AAC) {
LOG_ERROR(Audio_DSP, "Got wrong codec {}", static_cast<u16>(request.codec));
return {};
}
switch (request.cmd) {
case DecoderCommand::Init: {
return Initalize(request);
break;
}
case DecoderCommand::Decode: {
return Decode(request);
break;
}
case DecoderCommand::Unknown: {
BinaryResponse response;
std::memcpy(&response, &request, sizeof(response));
response.unknown1 = 0x0;
return response;
break;
}
default:
LOG_ERROR(Audio_DSP, "Got unknown binary request: {}", static_cast<u16>(request.cmd));
return {};
break;
}
}
std::optional<BinaryResponse> AACDecoder::Impl::Initalize(const BinaryRequest& request) {
if (initalized) {
Clear();
}
if (!have_ffmpeg_dl) {
return {};
}
av_packet = av_packet_alloc_dl();
codec = avcodec_find_decoder_dl(AV_CODEC_ID_AAC);
if (!codec) {
LOG_ERROR(Audio_DSP, "Codec not found\n");
return {};
}
parser = av_parser_init_dl(codec->id);
if (!parser) {
LOG_ERROR(Audio_DSP, "Parser not found\n");
return {};
}
av_context = avcodec_alloc_context3_dl(codec);
if (!av_context) {
LOG_ERROR(Audio_DSP, "Could not allocate audio codec context\n");
return {};
}
if (avcodec_open2_dl(av_context, codec, NULL) < 0) {
LOG_ERROR(Audio_DSP, "Could not open codec\n");
return {};
}
initalized = true;
BinaryResponse response;
std::memcpy(&response, &request, sizeof(response));
response.unknown1 = 0x0;
return response;
}
void AACDecoder::Impl::Clear() {
if (!have_ffmpeg_dl) {
return;
}
avcodec_free_context_dl(&av_context);
av_parser_close_dl(parser);
av_frame_free_dl(&decoded_frame);
av_packet_free_dl(&av_packet);
}
std::optional<BinaryResponse> AACDecoder::Impl::Decode(const BinaryRequest& request) {
if (!initalized) {
LOG_DEBUG(Audio_DSP, "Decoder not initalized");
// This is a hack to continue games that are not compiled with the aac codec
BinaryResponse response;
response.codec = request.codec;
response.cmd = request.cmd;
response.num_channels = 2;
response.num_samples = 1024;
response.size = request.size;
return response;
}
if (request.src_addr < Memory::FCRAM_PADDR) {
LOG_ERROR(Audio_DSP, "Got out of bounds src_addr {:08x}", request.src_addr);
return {};
}
u8* data = memory.GetFCRAMPointer(request.src_addr - Memory::FCRAM_PADDR);
std::array<std::vector<u8>, 2> out_streams;
std::size_t data_size = request.size;
while (data_size > 0) {
if (!decoded_frame) {
if (!(decoded_frame = av_frame_alloc_dl())) {
LOG_ERROR(Audio_DSP, "Could not allocate audio frame");
return {};
}
}
int ret = av_parser_parse2_dl(parser, av_context, &av_packet->data, &av_packet->size, data,
data_size, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
if (ret < 0) {
LOG_ERROR(Audio_DSP, "Error while parsing");
return {};
}
data += ret;
data_size -= ret;
ret = avcodec_send_packet_dl(av_context, av_packet);
if (ret < 0) {
LOG_ERROR(Audio_DSP, "Error submitting the packet to the decoder");
return {};
}
if (av_packet->size) {
while (ret >= 0) {
ret = avcodec_receive_frame_dl(av_context, decoded_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
else if (ret < 0) {
LOG_ERROR(Audio_DSP, "Error during decoding");
return {};
}
int bytes_per_sample = av_get_bytes_per_sample_dl(av_context->sample_fmt);
if (bytes_per_sample < 0) {
LOG_ERROR(Audio_DSP, "Failed to calculate data size");
return {};
}
ASSERT(decoded_frame->channels == out_streams.size());
std::size_t size = bytes_per_sample * (decoded_frame->nb_samples);
// FFmpeg converts to 32 signed floating point PCM, we need s32 PCM so we need to
// convert it
f32 val_float;
for (std::size_t current_pos(0); current_pos < size;) {
for (std::size_t channel(0); channel < out_streams.size(); channel++) {
std::memcpy(&val_float, decoded_frame->data[0] + current_pos,
sizeof(val_float));
s16 val = static_cast<s16>(0x7FFF * val_float);
out_streams[channel].push_back(val & 0xFF);
out_streams[channel].push_back(val >> 8);
}
current_pos += sizeof(val_float);
}
}
}
}
if (request.dst_addr_ch0 < Memory::FCRAM_PADDR) {
LOG_ERROR(Audio_DSP, "Got out of bounds dst_addr_ch0 {:08x}", request.dst_addr_ch0);
return {};
}
std::memcpy(memory.GetFCRAMPointer(request.dst_addr_ch0 - Memory::FCRAM_PADDR),
out_streams[0].data(), out_streams[0].size());
if (request.dst_addr_ch1 < Memory::FCRAM_PADDR) {
LOG_ERROR(Audio_DSP, "Got out of bounds dst_addr_ch1 {:08x}", request.dst_addr_ch1);
return {};
}
std::memcpy(memory.GetFCRAMPointer(request.dst_addr_ch1 - Memory::FCRAM_PADDR),
out_streams[1].data(), out_streams[1].size());
BinaryResponse response;
response.codec = request.codec;
response.cmd = request.cmd;
response.num_channels = 2;
response.num_samples = decoded_frame->nb_samples;
response.size = request.size;
return response;
}
AACDecoder::AACDecoder(Memory::MemorySystem& memory) : impl(std::make_unique<Impl>(memory)) {}
AACDecoder::~AACDecoder() = default;
std::optional<BinaryResponse> AACDecoder::ProcessRequest(const BinaryRequest& request) {
return impl->ProcessRequest(request);
}
} // namespace AudioCore::HLE

View file

@ -0,0 +1,22 @@
// Copyright 2018 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#include "audio_core/hle/decoder.h"
namespace AudioCore::HLE {
class AACDecoder final : public DecoderBase {
public:
AACDecoder(Memory::MemorySystem& memory);
~AACDecoder() override;
std::optional<BinaryResponse> ProcessRequest(const BinaryRequest& request) override;
private:
class Impl;
std::unique_ptr<Impl> impl;
};
} // namespace AudioCore::HLE

View file

@ -0,0 +1,36 @@
// Copyright 2018 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include "audio_core/hle/decoder.h"
namespace AudioCore::HLE {
NullDecoder::NullDecoder() {}
NullDecoder::~NullDecoder() = default;
std::optional<BinaryResponse> NullDecoder::ProcessRequest(const BinaryRequest& request) {
BinaryResponse response;
switch (request.cmd) {
case DecoderCommand::Init:
case DecoderCommand::Unknown:
std::memcpy(&response, &request, sizeof(response));
response.unknown1 = 0x0;
return response;
break;
case DecoderCommand::Decode:
response.codec = request.codec;
response.cmd = DecoderCommand::Decode;
response.num_channels = 2; // Just assume stereo here
response.size = request.size;
response.num_samples = 1024; // Just assume 1024 here
return response;
break;
default:
LOG_ERROR(Audio_DSP, "Got unknown binary request: {}", static_cast<u16>(request.cmd));
return {};
break;
}
};
} // namespace AudioCore::HLE

View file

@ -0,0 +1,68 @@
// Copyright 2018 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#include <memory>
#include <optional>
#include <vector>
#include "common/common_types.h"
#include "common/swap.h"
#include "core/core.h"
namespace AudioCore::HLE {
enum class DecoderCommand : u16 {
Init,
Decode,
Unknown,
};
enum class DecoderCodec : u16 {
None,
AAC,
};
struct BinaryRequest {
enum_le<DecoderCodec> codec =
DecoderCodec::None; // this is a guess. until now only 0x1 was observed here
enum_le<DecoderCommand> cmd = DecoderCommand::Init;
u32_le fixed = 0;
u32_le src_addr = 0;
u32_le size = 0;
u32_le dst_addr_ch0 = 0;
u32_le dst_addr_ch1 = 0;
u32_le unknown1 = 0;
u32_le unknown2 = 0;
};
static_assert(sizeof(BinaryRequest) == 32, "Unexpected struct size for BinaryRequest");
struct BinaryResponse {
enum_le<DecoderCodec> codec =
DecoderCodec::None; // this could be something else. until now only 0x1 was observed here
enum_le<DecoderCommand> cmd = DecoderCommand::Init;
u32_le unknown1 = 0;
u32_le unknown2 = 0;
u32_le num_channels = 0; // this is a guess, so far I only observed 2 here
u32_le size = 0;
u32_le unknown3 = 0;
u32_le unknown4 = 0;
u32_le num_samples = 0; // this is a guess, so far I only observed 1024 here
};
static_assert(sizeof(BinaryResponse) == 32, "Unexpected struct size for BinaryResponse");
class DecoderBase {
public:
virtual ~DecoderBase(){};
virtual std::optional<BinaryResponse> ProcessRequest(const BinaryRequest& request) = 0;
};
class NullDecoder final : public DecoderBase {
public:
NullDecoder();
~NullDecoder() override;
std::optional<BinaryResponse> ProcessRequest(const BinaryRequest& request) override;
};
} // namespace AudioCore::HLE

View file

@ -0,0 +1,213 @@
// Copyright 2018 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#ifdef _WIN32
#include <windows.h>
#endif
#include "common/logging/log.h"
extern "C" {
#include "libavcodec/avcodec.h"
}
#ifdef _WIN32
template <typename T>
struct FuncDL {
FuncDL() = default;
FuncDL(HMODULE dll, const char* name) {
ptr_function = nullptr;
if (dll) {
*(void**)&ptr_function = (void*)GetProcAddress(dll, name);
}
}
operator T*() const {
return ptr_function;
}
operator bool() const {
return ptr_function != nullptr;
}
T* ptr_function = nullptr;
;
};
FuncDL<int(AVSampleFormat)> av_get_bytes_per_sample_dl;
FuncDL<AVFrame*(void)> av_frame_alloc_dl;
FuncDL<void(AVFrame**)> av_frame_free_dl;
FuncDL<AVCodecContext*(const AVCodec*)> avcodec_alloc_context3_dl;
FuncDL<void(AVCodecContext**)> avcodec_free_context_dl;
FuncDL<int(AVCodecContext*, const AVCodec*, AVDictionary**)> avcodec_open2_dl;
FuncDL<AVPacket*(void)> av_packet_alloc_dl;
FuncDL<void(AVPacket**)> av_packet_free_dl;
FuncDL<AVCodec*(AVCodecID)> avcodec_find_decoder_dl;
FuncDL<int(AVCodecContext*, const AVPacket*)> avcodec_send_packet_dl;
FuncDL<int(AVCodecContext*, AVFrame*)> avcodec_receive_frame_dl;
FuncDL<AVCodecParserContext*(int)> av_parser_init_dl;
FuncDL<int(AVCodecParserContext*, AVCodecContext*, uint8_t**, int*, const uint8_t*, int, int64_t,
int64_t, int64_t)>
av_parser_parse2_dl;
FuncDL<void(AVCodecParserContext*)> av_parser_close_dl;
bool InitFFmpegDL() {
HMODULE dll_util = nullptr;
dll_util = LoadLibrary("avutil-56.dll");
if (!dll_util) {
DWORD errorMessageID = GetLastError();
LPSTR messageBuffer = nullptr;
size_t size =
FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPSTR)&messageBuffer, 0, NULL);
std::string message(messageBuffer, size);
// Free the buffer.
LocalFree(messageBuffer);
LOG_ERROR(Audio_DSP, "Could not load avcodec-58.dll: {}", message);
return false;
}
HMODULE dll_codec = nullptr;
dll_codec = LoadLibrary("avcodec-58.dll");
if (!dll_codec) {
DWORD errorMessageID = GetLastError();
LPSTR messageBuffer = nullptr;
size_t size =
FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPSTR)&messageBuffer, 0, NULL);
std::string message(messageBuffer, size);
// Free the buffer.
LocalFree(messageBuffer);
LOG_ERROR(Audio_DSP, "Could not load avcodec-58.dll: {}", message);
return false;
}
av_get_bytes_per_sample_dl = FuncDL<int(AVSampleFormat)>(dll_util, "av_get_bytes_per_sample");
if (!av_get_bytes_per_sample_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_get_bytes_per_sample");
return false;
}
av_frame_alloc_dl = FuncDL<AVFrame*()>(dll_util, "av_frame_alloc");
if (!av_frame_alloc_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_frame_alloc");
return false;
}
av_frame_free_dl = FuncDL<void(AVFrame**)>(dll_util, "av_frame_free");
if (!av_frame_free_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_frame_free");
return false;
}
avcodec_alloc_context3_dl =
FuncDL<AVCodecContext*(const AVCodec*)>(dll_codec, "avcodec_alloc_context3");
if (!avcodec_alloc_context3_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_alloc_context3");
return false;
}
avcodec_free_context_dl = FuncDL<void(AVCodecContext**)>(dll_codec, "avcodec_free_context");
if (!av_get_bytes_per_sample_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_free_context");
return false;
}
avcodec_open2_dl =
FuncDL<int(AVCodecContext*, const AVCodec*, AVDictionary**)>(dll_codec, "avcodec_open2");
if (!avcodec_open2_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_open2");
return false;
}
av_packet_alloc_dl = FuncDL<AVPacket*(void)>(dll_codec, "av_packet_alloc");
if (!av_packet_alloc_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_packet_alloc");
return false;
}
av_packet_free_dl = FuncDL<void(AVPacket**)>(dll_codec, "av_packet_free");
if (!av_packet_free_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_packet_free");
return false;
}
avcodec_find_decoder_dl = FuncDL<AVCodec*(AVCodecID)>(dll_codec, "avcodec_find_decoder");
if (!avcodec_find_decoder_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_find_decoder");
return false;
}
avcodec_send_packet_dl =
FuncDL<int(AVCodecContext*, const AVPacket*)>(dll_codec, "avcodec_send_packet");
if (!avcodec_send_packet_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_send_packet");
return false;
}
avcodec_receive_frame_dl =
FuncDL<int(AVCodecContext*, AVFrame*)>(dll_codec, "avcodec_receive_frame");
if (!avcodec_receive_frame_dl) {
LOG_ERROR(Audio_DSP, "Can not load function avcodec_receive_frame");
return false;
}
av_parser_init_dl = FuncDL<AVCodecParserContext*(int)>(dll_codec, "av_parser_init");
if (!av_parser_init_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_parser_init");
return false;
}
av_parser_parse2_dl =
FuncDL<int(AVCodecParserContext*, AVCodecContext*, uint8_t**, int*, const uint8_t*, int,
int64_t, int64_t, int64_t)>(dll_codec, "av_parser_parse2");
if (!av_parser_parse2_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_parser_parse2");
return false;
}
av_parser_close_dl = FuncDL<void(AVCodecParserContext*)>(dll_codec, "av_parser_close");
if (!av_parser_close_dl) {
LOG_ERROR(Audio_DSP, "Can not load function av_parser_close");
return false;
}
return true;
}
#endif // _Win32
#if defined(__APPLE__) || defined(__linux__)
// No dynamic loading for Unix and Apple
const auto av_get_bytes_per_sample_dl = &av_get_bytes_per_sample;
const auto av_frame_alloc_dl = &av_frame_alloc;
const auto av_frame_free_dl = &av_frame_free;
const auto avcodec_alloc_context3_dl = &avcodec_alloc_context3;
const auto avcodec_free_context_dl = &avcodec_free_context;
const auto avcodec_open2_dl = &avcodec_open2;
const auto av_packet_alloc_dl = &av_packet_alloc;
const auto av_packet_free_dl = &av_packet_free;
const auto avcodec_find_decoder_dl = &avcodec_find_decoder;
const auto avcodec_send_packet_dl = &avcodec_send_packet;
const auto avcodec_receive_frame_dl = &avcodec_receive_frame;
const auto av_parser_init_dl = &av_parser_init;
const auto av_parser_parse2_dl = &av_parser_parse2;
const auto av_parser_close_dl = &av_parser_close;
bool InitFFmpegDL() {
return true;
}
#endif // defined(__APPLE__) || defined(__linux__)

View file

@ -3,7 +3,11 @@
// Refer to the license.txt file included.
#include "audio_core/audio_types.h"
#ifdef HAVE_FFMPEG
#include "audio_core/hle/aac_decoder.h"
#endif
#include "audio_core/hle/common.h"
#include "audio_core/hle/decoder.h"
#include "audio_core/hle/hle.h"
#include "audio_core/hle/mixers.h"
#include "audio_core/hle/shared_memory.h"
@ -69,6 +73,8 @@ private:
DspHle& parent;
Core::TimingEventType* tick_event;
std::unique_ptr<HLE::DecoderBase> decoder;
std::weak_ptr<DSP_DSP> dsp_dsp;
};
@ -79,6 +85,13 @@ DspHle::Impl::Impl(DspHle& parent_, Memory::MemorySystem& memory) : parent(paren
source.SetMemory(memory);
}
#ifdef HAVE_FFMPEG
decoder = std::make_unique<HLE::AACDecoder>(memory);
#else
LOG_WARNING(Audio_DSP, "FFmpeg missing, this could lead to missing audio");
decoder = std::make_unique<HLE::NullDecoder>();
#endif // HAVE_FFMPEG
Core::Timing& timing = Core::System::GetInstance().CoreTiming();
tick_event =
timing.RegisterEvent("AudioCore::DspHle::tick_event", [this](u64, s64 cycles_late) {
@ -215,6 +228,28 @@ void DspHle::Impl::PipeWrite(DspPipe pipe_number, const std::vector<u8>& buffer)
return;
}
case DspPipe::Binary: {
// TODO(B3N30): Make this async, and signal the interrupt
HLE::BinaryRequest request;
if (sizeof(request) != buffer.size()) {
LOG_CRITICAL(Audio_DSP, "got binary pipe with wrong size {}", buffer.size());
UNIMPLEMENTED();
return;
}
std::memcpy(&request, buffer.data(), buffer.size());
if (request.codec != HLE::DecoderCodec::AAC) {
LOG_CRITICAL(Audio_DSP, "got unknown codec {}", static_cast<u16>(request.codec));
UNIMPLEMENTED();
return;
}
std::optional<HLE::BinaryResponse> response = decoder->ProcessRequest(request);
if (response) {
const HLE::BinaryResponse& value = *response;
pipe_data[static_cast<u32>(pipe_number)].resize(sizeof(value));
std::memcpy(pipe_data[static_cast<u32>(pipe_number)].data(), &value, sizeof(value));
}
break;
}
default:
LOG_CRITICAL(Audio_DSP, "pipe_number = {} unimplemented",
static_cast<std::size_t>(pipe_number));