Fix Vi managed and stray layers open/close/destroy (#3438)

* Fix Vi managed and stray layers open/close/destroy

* OpenLayer should set the state to ManagedOpened

Ryujinx.HLE/HOS/Services/Am/AppletAE/AllSystemAppletProxiesService/SystemAppletProxy/ISelfController.cs

@@ -217,7 +217,7 @@ namespace Ryujinx.HLE.HOS.Services.Am.AppletAE.AllSystemAppletProxiesService.Sys
         // CreateManagedDisplayLayer() -> u64
         public ResultCode CreateManagedDisplayLayer(ServiceCtx context)
         {
-            context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long layerId);
+            context.Device.System.SurfaceFlinger.CreateLayer(out long layerId, _pid);
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
             context.ResponseData.Write(layerId);
@@ -238,8 +238,8 @@ namespace Ryujinx.HLE.HOS.Services.Am.AppletAE.AllSystemAppletProxiesService.Sys
         // CreateManagedDisplaySeparableLayer() -> (u64, u64)
         public ResultCode CreateManagedDisplaySeparableLayer(ServiceCtx context)
         {
-            context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long displayLayerId);
+            context.Device.System.SurfaceFlinger.CreateLayer(out long displayLayerId, _pid);
-            context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long recordingLayerId);
+            context.Device.System.SurfaceFlinger.CreateLayer(out long recordingLayerId, _pid);
             context.Device.System.SurfaceFlinger.SetRenderLayer(displayLayerId);
 
             context.ResponseData.Write(displayLayerId);

Ryujinx.HLE/HOS/Services/SurfaceFlinger/LayerState.cs

@@ -0,0 +1,10 @@
+namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
+{
+    enum LayerState
+    {
+        NotInitialized,
+        ManagedClosed,
+        ManagedOpened,
+        Stray
+    }
+}

Ryujinx.HLE/HOS/Services/SurfaceFlinger/SurfaceFlinger.cs

@@ -11,6 +11,8 @@ using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 {
+    using ResultCode = Ryujinx.HLE.HOS.Services.Vi.ResultCode;
+
     class SurfaceFlinger : IConsumerListener, IDisposable
     {
         private const int TargetFps = 60;
@@ -45,6 +47,7 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
             public BufferItemConsumer     Consumer;
             public BufferQueueCore        Core;
             public ulong                  Owner;
+            public LayerState             State;
         }
 
         private class TextureCallbackInformation
@@ -92,24 +95,7 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
             }
         }
 
-        public IGraphicBufferProducer OpenLayer(ulong pid, long layerId)
+        public IGraphicBufferProducer CreateLayer(out long layerId, ulong pid, LayerState initialState = LayerState.ManagedClosed)
-        {
-            bool needCreate;
-
-            lock (Lock)
-            {
-                needCreate = GetLayerByIdLocked(layerId) == null;
-            }
-
-            if (needCreate)
-            {
-                CreateLayerFromId(pid, layerId);
-            }
-
-            return GetProducerByLayerId(layerId);
-        }
-
-        public IGraphicBufferProducer CreateLayer(ulong pid, out long layerId)
         {
             layerId = 1;
 
@@ -124,12 +110,12 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                 }
             }
 
-            CreateLayerFromId(pid, layerId);
+            CreateLayerFromId(pid, layerId, initialState);
 
             return GetProducerByLayerId(layerId);
         }
 
-        private void CreateLayerFromId(ulong pid, long layerId)
+        private void CreateLayerFromId(ulong pid, long layerId, LayerState initialState)
         {
             lock (Lock)
             {
@@ -148,26 +134,114 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                     Producer         = producer,
                     Consumer         = new BufferItemConsumer(_device, consumer, 0, -1, false, this),
                     Core             = core,
-                    Owner            = pid
+                    Owner            = pid,
+                    State            = initialState
                 });
             }
         }
 
-        public bool CloseLayer(long layerId)
+        public ResultCode OpenLayer(ulong pid, long layerId, out IBinder producer)
+        {
+            Layer layer = GetLayerByIdLocked(layerId);
+
+            if (layer == null || layer.State != LayerState.ManagedClosed)
+            {
+                producer = null;
+
+                return ResultCode.InvalidArguments;
+            }
+
+            layer.State = LayerState.ManagedOpened;
+            producer = layer.Producer;
+
+            return ResultCode.Success;
+        }
+
+        public ResultCode CloseLayer(long layerId)
         {
             lock (Lock)
             {
                 Layer layer = GetLayerByIdLocked(layerId);
 
-                if (layer != null)
+                if (layer == null)
                 {
-                    HOSBinderDriverServer.UnregisterBinderObject(layer.ProducerBinderId);
+                    Logger.Error?.Print(LogClass.SurfaceFlinger, $"Failed to close layer {layerId}");
+
+                    return ResultCode.InvalidValue;
                 }
 
-                bool removed = _layers.Remove(layerId);
+                CloseLayer(layerId, layer);
 
+                return ResultCode.Success;
+            }
+        }
+
+        public ResultCode DestroyManagedLayer(long layerId)
+        {
+            lock (Lock)
+            {
+                Layer layer = GetLayerByIdLocked(layerId);
+
+                if (layer == null)
+                {
+                    Logger.Error?.Print(LogClass.SurfaceFlinger, $"Failed to destroy managed layer {layerId} (not found)");
+
+                    return ResultCode.InvalidValue;
+                }
+
+                if (layer.State != LayerState.ManagedClosed && layer.State != LayerState.ManagedOpened)
+                {
+                    Logger.Error?.Print(LogClass.SurfaceFlinger, $"Failed to destroy managed layer {layerId} (permission denied)");
+
+                    return ResultCode.PermissionDenied;
+                }
+
+                HOSBinderDriverServer.UnregisterBinderObject(layer.ProducerBinderId);
+
+                if (_layers.Remove(layerId) && layer.State == LayerState.ManagedOpened)
+                {
+                    CloseLayer(layerId, layer);
+                }
+
+                return ResultCode.Success;
+            }
+        }
+
+        public ResultCode DestroyStrayLayer(long layerId)
+        {
+            lock (Lock)
+            {
+                Layer layer = GetLayerByIdLocked(layerId);
+
+                if (layer == null)
+                {
+                    Logger.Error?.Print(LogClass.SurfaceFlinger, $"Failed to destroy stray layer {layerId} (not found)");
+
+                    return ResultCode.InvalidValue;
+                }
+
+                if (layer.State != LayerState.Stray)
+                {
+                    Logger.Error?.Print(LogClass.SurfaceFlinger, $"Failed to destroy stray layer {layerId} (permission denied)");
+
+                    return ResultCode.PermissionDenied;
+                }
+
+                HOSBinderDriverServer.UnregisterBinderObject(layer.ProducerBinderId);
+
+                if (_layers.Remove(layerId))
+                {
+                    CloseLayer(layerId, layer);
+                }
+
+                return ResultCode.Success;
+            }
+        }
+
+        private void CloseLayer(long layerId, Layer layer)
+        {
             // If the layer was removed and the current in use, we need to change the current layer in use.
-                if (removed && RenderLayerId == layerId)
+            if (RenderLayerId == layerId)
             {
                 // If no layer is availaible, reset to default value.
                 if (_layers.Count == 0)
@@ -180,7 +254,9 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                 }
             }
 
-                return removed;
+            if (layer.State == LayerState.ManagedOpened)
+            {
+                layer.State = LayerState.ManagedClosed;
             }
         }
 

Ryujinx.HLE/HOS/Services/Vi/RootService/ApplicationDisplayService/IManagerDisplayService.cs

@@ -35,7 +35,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService.ApplicationDisplayService
 
             ulong pid = context.Device.System.AppletState.AppletResourceUserIds.GetData<ulong>((int)appletResourceUserId);
 
-            context.Device.System.SurfaceFlinger.CreateLayer(pid, out long layerId);
+            context.Device.System.SurfaceFlinger.CreateLayer(out long layerId, pid);
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
             context.ResponseData.Write(layerId);
@@ -49,9 +49,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService.ApplicationDisplayService
         {
             long layerId = context.RequestData.ReadInt64();
 
-            context.Device.System.SurfaceFlinger.CloseLayer(layerId);
+            return context.Device.System.SurfaceFlinger.DestroyManagedLayer(layerId);
-
-            return ResultCode.Success;
         }
 
         [CommandHipc(2012)] // 7.0.0+

Ryujinx.HLE/HOS/Services/Vi/RootService/IApplicationDisplayService.cs

@@ -237,7 +237,12 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
             long  userId    = context.RequestData.ReadInt64();
             ulong parcelPtr = context.Request.ReceiveBuff[0].Position;
 
-            IBinder producer = context.Device.System.SurfaceFlinger.OpenLayer(context.Request.HandleDesc.PId, layerId);
+            ResultCode result = context.Device.System.SurfaceFlinger.OpenLayer(context.Request.HandleDesc.PId, layerId, out IBinder producer);
+
+            if (result != ResultCode.Success)
+            {
+                return result;
+            }
 
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
@@ -260,9 +265,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
         {
             long layerId = context.RequestData.ReadInt64();
 
-            context.Device.System.SurfaceFlinger.CloseLayer(layerId);
+            return context.Device.System.SurfaceFlinger.CloseLayer(layerId);
-
-            return ResultCode.Success;
         }
 
         [CommandHipc(2030)]
@@ -275,7 +278,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
             ulong parcelPtr = context.Request.ReceiveBuff[0].Position;
 
             // TODO: support multi display.
-            IBinder producer = context.Device.System.SurfaceFlinger.CreateLayer(0, out long layerId);
+            IBinder producer = context.Device.System.SurfaceFlinger.CreateLayer(out long layerId, 0, LayerState.Stray);
 
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
@@ -299,9 +302,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
         {
             long layerId = context.RequestData.ReadInt64();
 
-            context.Device.System.SurfaceFlinger.CloseLayer(layerId);
+            return context.Device.System.SurfaceFlinger.DestroyStrayLayer(layerId);
-
-            return ResultCode.Success;
         }
 
         [CommandHipc(2101)]