Do not allow render targets not explicitly written by the fragment shader to be modified (#3063)

* Do not allow render targets not explicitly written by the fragment shader to be modified

* Shader cache version bump

* Remove blank lines

* Avoid redundant color mask updates

* HostShaderCacheEntry can be null

* Avoid more redundant glColorMask calls

* nit: Mask -> Masks

* Fix currentComponentMask

* More efficient way to update _currentComponentMasks

Ryujinx.Graphics.OpenGL/Pipeline.cs

@@ -53,7 +53,9 @@ namespace Ryujinx.Graphics.OpenGL
         private ClipOrigin _clipOrigin;
         private ClipDepthMode _clipDepthMode;
 
-        private readonly uint[] _componentMasks;
+        private uint _fragmentOutputMap;
+        private uint _componentMasks;
+        private uint _currentComponentMasks;
 
         private uint _scissorEnables;
 
@@ -73,12 +75,8 @@ namespace Ryujinx.Graphics.OpenGL
             _clipOrigin = ClipOrigin.LowerLeft;
             _clipDepthMode = ClipDepthMode.NegativeOneToOne;
 
-            _componentMasks = new uint[Constants.MaxRenderTargets];
-
-            for (int index = 0; index < Constants.MaxRenderTargets; index++)
-            {
-                _componentMasks[index] = 0xf;
-            }
+            _fragmentOutputMap = uint.MaxValue;
+            _componentMasks = uint.MaxValue;
 
             var defaultScale = new Vector4<float> { X = 1f, Y = 0f, Z = 0f, W = 0f };
             new Span<Vector4<float>>(_renderScale).Fill(defaultScale);
@@ -1001,18 +999,30 @@ namespace Ryujinx.Graphics.OpenGL
 
         public void SetProgram(IProgram program)
         {
-            _program = (Program)program;
+            Program prg = (Program)program;
 
             if (_tfEnabled)
             {
                 GL.EndTransformFeedback();
-                _program.Bind();
+                prg.Bind();
                 GL.BeginTransformFeedback(_tfTopology);
             }
             else
             {
-                _program.Bind();
+                prg.Bind();
             }
+
+            if (prg.HasFragmentShader && _fragmentOutputMap != (uint)prg.FragmentOutputMap)
+            {
+                _fragmentOutputMap = (uint)prg.FragmentOutputMap;
+
+                for (int index = 0; index < Constants.MaxRenderTargets; index++)
+                {
+                    RestoreComponentMask(index, force: false);
+                }
+            }
+
+            _program = prg;
         }
 
         public void SetRasterizerDiscard(bool discard)
@@ -1037,11 +1047,13 @@ namespace Ryujinx.Graphics.OpenGL
 
         public void SetRenderTargetColorMasks(ReadOnlySpan<uint> componentMasks)
         {
+            _componentMasks = 0;
+
             for (int index = 0; index < componentMasks.Length; index++)
             {
-                _componentMasks[index] = componentMasks[index];
+                _componentMasks |= componentMasks[index] << (index * 4);
 
-                RestoreComponentMask(index);
+                RestoreComponentMask(index, force: false);
             }
         }
 
@@ -1436,18 +1448,34 @@ namespace Ryujinx.Graphics.OpenGL
             }
         }
 
-        public void RestoreComponentMask(int index)
+        public void RestoreComponentMask(int index, bool force = true)
         {
             // If the bound render target is bgra, swap the red and blue masks.
             uint redMask = _fpIsBgra[index].X == 0 ? 1u : 4u;
             uint blueMask = _fpIsBgra[index].X == 0 ? 4u : 1u;
 
+            int shift = index * 4;
+            uint componentMask = _componentMasks & _fragmentOutputMap;
+            uint checkMask = 0xfu << shift;
+            uint componentMaskAtIndex = componentMask & checkMask;
+
+            if (!force && componentMaskAtIndex == (_currentComponentMasks & checkMask))
+            {
+                return;
+            }
+
+            componentMask >>= shift;
+            componentMask &= 0xfu;
+
             GL.ColorMask(
                 index,
-                (_componentMasks[index] & redMask) != 0,
-                (_componentMasks[index] & 2u) != 0,
-                (_componentMasks[index] & blueMask) != 0,
-                (_componentMasks[index] & 8u) != 0);
+                (componentMask & redMask) != 0,
+                (componentMask & 2u) != 0,
+                (componentMask & blueMask) != 0,
+                (componentMask & 8u) != 0);
+
+            _currentComponentMasks &= ~checkMask;
+            _currentComponentMasks |= componentMaskAtIndex;
         }
 
         public void RestoreScissor0Enable()

Ryujinx.Graphics.OpenGL/Program.cs

@@ -1,11 +1,8 @@
 using OpenTK.Graphics.OpenGL;
 using Ryujinx.Common.Logging;
 using Ryujinx.Graphics.GAL;
-using Ryujinx.Graphics.Shader.CodeGen.Glsl;
 using System;
 using System.Buffers.Binary;
-using System.Collections.Generic;
-using System.Linq;
 
 namespace Ryujinx.Graphics.OpenGL
 {
@@ -29,7 +26,10 @@ namespace Ryujinx.Graphics.OpenGL
         private ProgramLinkStatus _status = ProgramLinkStatus.Incomplete;
         private IShader[] _shaders;
 
-        public Program(IShader[] shaders)
+        public bool HasFragmentShader;
+        public int FragmentOutputMap { get; }
+
+        public Program(IShader[] shaders, int fragmentOutputMap)
         {
             Handle = GL.CreateProgram();
 
@@ -37,17 +37,23 @@ namespace Ryujinx.Graphics.OpenGL
 
             for (int index = 0; index < shaders.Length; index++)
             {
-                int shaderHandle = ((Shader)shaders[index]).Handle;
+                Shader shader = (Shader)shaders[index];
 
-                GL.AttachShader(Handle, shaderHandle);
+                if (shader.IsFragment)
+                {
+                    HasFragmentShader = true;
+                }
+
+                GL.AttachShader(Handle, shader.Handle);
             }
 
             GL.LinkProgram(Handle);
 
             _shaders = shaders;
+            FragmentOutputMap = fragmentOutputMap;
         }
 
-        public Program(ReadOnlySpan<byte> code)
+        public Program(ReadOnlySpan<byte> code, bool hasFragmentShader, int fragmentOutputMap)
         {
             BinaryFormat binaryFormat = (BinaryFormat)BinaryPrimitives.ReadInt32LittleEndian(code.Slice(code.Length - 4, 4));
 
@@ -60,6 +66,9 @@ namespace Ryujinx.Graphics.OpenGL
                     GL.ProgramBinary(Handle, binaryFormat, (IntPtr)ptr, code.Length - 4);
                 }
             }
+
+            HasFragmentShader = hasFragmentShader;
+            FragmentOutputMap = fragmentOutputMap;
         }
 
         public void Bind()

Ryujinx.Graphics.OpenGL/Renderer.cs

@@ -66,9 +66,9 @@ namespace Ryujinx.Graphics.OpenGL
             return Buffer.Create(size);
         }
 
-        public IProgram CreateProgram(IShader[] shaders)
+        public IProgram CreateProgram(IShader[] shaders, ShaderInfo info)
         {
-            return new Program(shaders);
+            return new Program(shaders, info.FragmentOutputMap);
         }
 
         public ISampler CreateSampler(SamplerCreateInfo info)
@@ -202,9 +202,9 @@ namespace Ryujinx.Graphics.OpenGL
             _sync.Dispose();
         }
 
-        public IProgram LoadProgramBinary(byte[] programBinary)
+        public IProgram LoadProgramBinary(byte[] programBinary, bool hasFragmentShader, ShaderInfo info)
         {
-            return new Program(programBinary);
+            return new Program(programBinary, hasFragmentShader, info.FragmentOutputMap);
         }
 
         public void CreateSync(ulong id)

Ryujinx.Graphics.OpenGL/Shader.cs

@@ -7,6 +7,7 @@ namespace Ryujinx.Graphics.OpenGL
     class Shader : IShader
     {
         public int Handle { get; private set; }
+        public bool IsFragment { get; }
 
         public Shader(ShaderStage stage, string code)
         {
@@ -22,6 +23,7 @@ namespace Ryujinx.Graphics.OpenGL
             };
 
             Handle = GL.CreateShader(type);
+            IsFragment = stage == ShaderStage.Fragment;
 
             GL.ShaderSource(Handle, code);
             GL.CompileShader(Handle);