libSceVideodec2: Update structs to match newer firmwares (#3077)

* Update file_system.cpp

* libSceVideodec2 struct fixes

Our code was based on an old version of the libSceVideodec2 library. Based on what I've decompiled, these structs changed somewhere around firmware 6.50, and newer versions of the library have these flexible checks to accommodate both variants of the structs.

* Static assert for AvcPictureInfo struct

All the other Videodec2 structs have static asserts, might as well use one here too.

* Initialize new values

Set proper values for frameFormat and framePitchInBytes.
`frame->linesize[0]` appears to be in bytes already, I'm not sure if that means framePitch is being set wrong though.

src/core/libraries/kernel/file_system.cpp

@@ -1050,6 +1050,7 @@ void RegisterFileSystem(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("4wSze92BhLI", "libkernel", 1, "libkernel", 1, 1, sceKernelWrite);
     LIB_FUNCTION("+WRlkKjZvag", "libkernel", 1, "libkernel", 1, 1, readv);
     LIB_FUNCTION("YSHRBRLn2pI", "libkernel", 1, "libkernel", 1, 1, writev);
+    LIB_FUNCTION("kAt6VDbHmro", "libkernel", 1, "libkernel", 1, 1, sceKernelWritev);
     LIB_FUNCTION("Oy6IpwgtYOk", "libScePosix", 1, "libkernel", 1, 1, posix_lseek);
     LIB_FUNCTION("Oy6IpwgtYOk", "libkernel", 1, "libkernel", 1, 1, posix_lseek);
     LIB_FUNCTION("oib76F-12fk", "libkernel", 1, "libkernel", 1, 1, sceKernelLseek);

src/core/libraries/videodec/videodec2.cpp

@@ -140,7 +140,7 @@ s32 PS4_SYSV_ABI sceVideodec2Flush(OrbisVideodec2Decoder decoder,
         return ORBIS_VIDEODEC2_ERROR_ARGUMENT_POINTER;
     }
     if (frameBuffer->thisSize != sizeof(OrbisVideodec2FrameBuffer) ||
-        outputInfo->thisSize != sizeof(OrbisVideodec2OutputInfo)) {
+        (outputInfo->thisSize | 8) != sizeof(OrbisVideodec2OutputInfo)) {
         LOG_ERROR(Lib_Vdec2, "Invalid struct size");
         return ORBIS_VIDEODEC2_ERROR_STRUCT_SIZE;
     }
@@ -167,7 +167,7 @@ s32 PS4_SYSV_ABI sceVideodec2GetPictureInfo(const OrbisVideodec2OutputInfo* outp
         LOG_ERROR(Lib_Vdec2, "Invalid arguments");
         return ORBIS_VIDEODEC2_ERROR_ARGUMENT_POINTER;
     }
-    if (outputInfo->thisSize != sizeof(OrbisVideodec2OutputInfo)) {
+    if ((outputInfo->thisSize | 8) != sizeof(OrbisVideodec2OutputInfo)) {
         LOG_ERROR(Lib_Vdec2, "Invalid struct size");
         return ORBIS_VIDEODEC2_ERROR_STRUCT_SIZE;
     }
@@ -179,7 +179,7 @@ s32 PS4_SYSV_ABI sceVideodec2GetPictureInfo(const OrbisVideodec2OutputInfo* outp
     if (p1stPictureInfoOut) {
         OrbisVideodec2AvcPictureInfo* picInfo =
             static_cast<OrbisVideodec2AvcPictureInfo*>(p1stPictureInfoOut);
-        if (picInfo->thisSize != sizeof(OrbisVideodec2AvcPictureInfo)) {
+        if ((picInfo->thisSize | 16) != sizeof(OrbisVideodec2AvcPictureInfo)) {
             LOG_ERROR(Lib_Vdec2, "Invalid struct size");
             return ORBIS_VIDEODEC2_ERROR_STRUCT_SIZE;
         }

src/core/libraries/videodec/videodec2.h

@@ -73,8 +73,10 @@ struct OrbisVideodec2OutputInfo {
     u32 frameHeight;
     void* frameBuffer;
     u64 frameBufferSize;
+    u32 frameFormat;
+    u32 framePitchInBytes;
 };
-static_assert(sizeof(OrbisVideodec2OutputInfo) == 0x30);
+static_assert(sizeof(OrbisVideodec2OutputInfo) == 0x38);
 
 struct OrbisVideodec2FrameBuffer {
     u64 thisSize;

src/core/libraries/videodec/videodec2_avc.h

@@ -55,6 +55,23 @@ struct OrbisVideodec2AvcPictureInfo {
     u8 pic_struct;
     u8 field_pic_flag;
     u8 bottom_field_flag;
+
+    u8 sequenceParameterSetPresentFlag;
+    u8 pictureParameterSetPresentFlag;
+    u8 auDelimiterPresentFlag;
+    u8 endOfSequencePresentFlag;
+    u8 endOfStreamPresentFlag;
+    u8 fillerDataPresentFlag;
+    u8 pictureTimingSeiPresentFlag;
+    u8 bufferingPeriodSeiPresentFlag;
+
+    u8 constraint_set0_flag;
+    u8 constraint_set1_flag;
+    u8 constraint_set2_flag;
+    u8 constraint_set3_flag;
+    u8 constraint_set4_flag;
+    u8 constraint_set5_flag;
 };
+static_assert(sizeof(OrbisVideodec2AvcPictureInfo) == 0x78);
 
 } // namespace Libraries::Vdec2

src/core/libraries/videodec/videodec2_impl.cpp

@@ -48,6 +48,7 @@ s32 VdecDecoder::Decode(const OrbisVideodec2InputData& inputData,
     outputInfo.isValid = false;
     outputInfo.isErrorFrame = true;
     outputInfo.pictureCount = 0;
+    outputInfo.frameFormat = 0;
 
     if (!inputData.auData) {
         return ORBIS_VIDEODEC2_ERROR_ACCESS_UNIT_POINTER;
@@ -106,6 +107,7 @@ s32 VdecDecoder::Decode(const OrbisVideodec2InputData& inputData,
         outputInfo.frameWidth = frame->width;
         outputInfo.frameHeight = frame->height;
         outputInfo.framePitch = frame->linesize[0];
+        outputInfo.framePitchInBytes = frame->linesize[0];
         outputInfo.frameBufferSize = frameBuffer.frameBufferSize;
         outputInfo.frameBuffer = frameBuffer.frameBuffer;
 
@@ -144,6 +146,7 @@ s32 VdecDecoder::Flush(OrbisVideodec2FrameBuffer& frameBuffer,
     outputInfo.isValid = false;
     outputInfo.isErrorFrame = true;
     outputInfo.pictureCount = 0;
+    outputInfo.frameFormat = 0;
 
     AVFrame* frame = av_frame_alloc();
     if (!frame) {
@@ -175,6 +178,7 @@ s32 VdecDecoder::Flush(OrbisVideodec2FrameBuffer& frameBuffer,
         outputInfo.frameWidth = frame->width;
         outputInfo.frameHeight = frame->height;
         outputInfo.framePitch = frame->linesize[0];
+        outputInfo.framePitchInBytes = frame->linesize[0];
         outputInfo.frameBufferSize = frameBuffer.frameBufferSize;
         outputInfo.frameBuffer = frameBuffer.frameBuffer;