diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 9e080b97..cda73abe 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -31,6 +31,25 @@ class GaussianSplatRenderSystem CommandBuffer m_CommandBuffer; + // Keep track of the prepared splats for stereo rendering + public struct RenderItem + { + public GaussianSplatRenderer gs; + public Material displayMat; + public MaterialPropertyBlock mpb; + public int indexCount; + public int instanceCount; + public MeshTopology topology; + } + + public class PreparedRenderData + { + public Material matComposite; + public List renderItems = new(); + } + + private PreparedRenderData m_LastPreparedData; + public void RegisterSplat(GaussianSplatRenderer r) { if (m_Splats.Count == 0) @@ -104,10 +123,22 @@ public bool GatherSplatsForCamera(Camera cam) return true; } + // New optimized method that prepares everything once for stereo rendering + // This does the sorting and calculates view data, but doesn't actually render // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled - public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) + public PreparedRenderData PrepareSplats(Camera cam, CommandBuffer cmb) { + if (m_LastPreparedData == null) + { + m_LastPreparedData = new PreparedRenderData(); + } + else + { + m_LastPreparedData.renderItems.Clear(); + } + Material matComposite = null; + foreach (var kvp in m_ActiveSplats) { var gs = kvp.Item1; @@ -115,13 +146,13 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) matComposite = gs.m_MatComposite; var mpb = kvp.Item2; - // sort + // Sort the splats var matrix = gs.transform.localToWorldMatrix; if (gs.m_FrameCounter % gs.m_SortNthFrame == 0) gs.SortPoints(cmb, cam, matrix); ++gs.m_FrameCounter; - // cache view + // Prepare material and view data kvp.Item2.Clear(); Material displayMat = gs.m_RenderMode switch { @@ -134,11 +165,10 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) if (displayMat == null) continue; - gs.SetAssetDataOnMaterial(mpb); + // Set up everything except eye-specific parameters + gs.SetAssetDataOnMaterial(mpb, -1); // -1 for initial setup without eye index mpb.SetBuffer(GaussianSplatRenderer.Props.SplatChunks, gs.m_GpuChunks); - mpb.SetBuffer(GaussianSplatRenderer.Props.SplatViewData, gs.m_GpuView); - mpb.SetBuffer(GaussianSplatRenderer.Props.OrderBuffer, gs.m_GpuSortKeys); mpb.SetFloat(GaussianSplatRenderer.Props.SplatScale, gs.m_SplatScale); mpb.SetFloat(GaussianSplatRenderer.Props.SplatOpacityScale, gs.m_OpacityScale); @@ -148,11 +178,12 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) mpb.SetInteger(GaussianSplatRenderer.Props.DisplayIndex, gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugPointIndices ? 1 : 0); mpb.SetInteger(GaussianSplatRenderer.Props.DisplayChunks, gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugChunkBounds ? 1 : 0); + // Calculate view data once for stereo (will calculate for both eyes) cmb.BeginSample(s_ProfCalcView); gs.CalcViewData(cmb, cam); cmb.EndSample(s_ProfCalcView); - // draw + // Set up draw parameters int indexCount = 6; int instanceCount = gs.splatCount; MeshTopology topology = MeshTopology.Triangles; @@ -160,12 +191,45 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) indexCount = 36; if (gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugChunkBounds) instanceCount = gs.m_GpuChunksValid ? gs.m_GpuChunks.count : 0; + // Store the prepared data for rendering later + m_LastPreparedData.renderItems.Add(new RenderItem { gs = gs, displayMat = displayMat, mpb = mpb, indexCount = indexCount, instanceCount = instanceCount, topology = topology }); + } + + m_LastPreparedData.matComposite = matComposite; + return m_LastPreparedData; + } + + // New optimized method that just draws the prepared splats for a specific eye + // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled + public void RenderPreparedSplats(CommandBuffer cmb, int eyeIndex) + { + if (m_LastPreparedData == null || m_LastPreparedData.renderItems.Count == 0) + return; + + foreach (var item in m_LastPreparedData.renderItems) + { + // Set the eye index for this specific render + item.mpb.SetInteger(GaussianSplatRenderer.Props.EyeIndex, eyeIndex); + item.mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, (eyeIndex == -1) ? 0 : 1); + // Draw cmb.BeginSample(s_ProfDraw); - cmb.DrawProcedural(gs.m_GpuIndexBuffer, matrix, displayMat, 0, topology, indexCount, instanceCount, mpb); + cmb.DrawProcedural(item.gs.m_GpuIndexBuffer, item.gs.transform.localToWorldMatrix, item.displayMat, 0, item.topology, item.indexCount, item.instanceCount, item.mpb); cmb.EndSample(s_ProfDraw); } - return matComposite; + } + + // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled + public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex = -1) + { + // Prepare the splats (sort and calculate view data) + var renderData = PrepareSplats(cam, cmb); + + // Render the prepared splats + RenderPreparedSplats(cmb, eyeIndex); + + // Return the composite material + return renderData.matComposite; } // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled @@ -209,6 +273,7 @@ void OnPreCullCamera(Camera cam) m_CommandBuffer.EndSample(s_ProfCompose); m_CommandBuffer.ReleaseTemporaryRT(GaussianSplatRenderer.Props.GaussianSplatRT); } + } [ExecuteInEditMode] @@ -237,7 +302,6 @@ public enum RenderMode public bool m_SHOnly; [Range(1,30)] [Tooltip("Sort splats only every N frames")] public int m_SortNthFrame = 1; - public RenderMode m_RenderMode = RenderMode.Splats; [Range(1.0f,15.0f)] public float m_PointDisplaySize = 3.0f; @@ -297,6 +361,8 @@ internal static class Props public static readonly int SplatBitsValid = Shader.PropertyToID("_SplatBitsValid"); public static readonly int SplatFormat = Shader.PropertyToID("_SplatFormat"); public static readonly int SplatChunks = Shader.PropertyToID("_SplatChunks"); + public static readonly int EyeIndex = Shader.PropertyToID("_EyeIndex"); + public static readonly int IsStereo = Shader.PropertyToID("_IsStereo"); public static readonly int SplatChunkCount = Shader.PropertyToID("_SplatChunkCount"); public static readonly int SplatViewData = Shader.PropertyToID("_SplatViewData"); public static readonly int OrderBuffer = Shader.PropertyToID("_OrderBuffer"); @@ -328,6 +394,12 @@ internal static class Props public static readonly int SelectionMode = Shader.PropertyToID("_SelectionMode"); public static readonly int SplatPosMouseDown = Shader.PropertyToID("_SplatPosMouseDown"); public static readonly int SplatOtherMouseDown = Shader.PropertyToID("_SplatOtherMouseDown"); + public static readonly int ViewProjMatrixLeft = Shader.PropertyToID("_ViewProjMatrixLeft"); + public static readonly int ViewProjMatrixRight = Shader.PropertyToID("_ViewProjMatrixRight"); + public static readonly int MatrixMVLeft = Shader.PropertyToID("_MatrixMVLeft"); + public static readonly int MatrixMVRight = Shader.PropertyToID("_MatrixMVRight"); + public static readonly int MatrixProjLeft = Shader.PropertyToID("_MatrixProjLeft"); + public static readonly int MatrixProjRight = Shader.PropertyToID("_MatrixProjRight"); } [field: NonSerialized] public bool editModified { get; private set; } @@ -404,7 +476,8 @@ void CreateResourcesForAsset() m_GpuChunksValid = false; } - m_GpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, m_Asset.splatCount, kGpuViewDataSize); + // Double the size to hold both left and right eye data for stereo rendering + m_GpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, m_Asset.splatCount * 2, kGpuViewDataSize); m_GpuIndexBuffer = new GraphicsBuffer(GraphicsBuffer.Target.Index, 36, 2); // cube indices, most often we use only the first quad m_GpuIndexBuffer.SetData(new ushort[] @@ -509,7 +582,7 @@ void SetAssetDataOnCS(CommandBuffer cmb, KernelIndices kernel) cmb.SetComputeBufferParam(cs, kernelIndex, Props.SplatCutouts, m_GpuEditCutouts); } - internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat) + internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat, int eyeIndex) { mat.SetBuffer(Props.SplatPos, m_GpuPosData); mat.SetBuffer(Props.SplatOther, m_GpuOtherData); @@ -517,6 +590,15 @@ internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat) mat.SetTexture(Props.SplatColor, m_GpuColorData); mat.SetBuffer(Props.SplatSelectedBits, m_GpuEditSelected ?? m_GpuPosData); mat.SetBuffer(Props.SplatDeletedBits, m_GpuEditDeleted ?? m_GpuPosData); + if (eyeIndex != -1) + { + mat.SetInteger(Props.EyeIndex, eyeIndex); + mat.SetInteger(Props.IsStereo, 1); + } + else + { + mat.SetInteger(Props.IsStereo, 0); + } mat.SetInt(Props.SplatBitsValid, m_GpuEditSelected != null && m_GpuEditDeleted != null ? 1 : 0); uint format = (uint)m_Asset.posFormat | ((uint)m_Asset.scaleFormat << 8) | ((uint)m_Asset.shFormat << 16); mat.SetInteger(Props.SplatFormat, (int)format); @@ -597,6 +679,33 @@ internal void CalcViewData(CommandBuffer cmb, Camera cam) cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixMV, matView * matO2W); cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixObjectToWorld, matO2W); cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixWorldToObject, matW2O); + bool isStereo = XRSettings.enabled && cam.stereoEnabled && + (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview) && + !Application.isEditor; + + if (isStereo) + { + Matrix4x4 stereoViewLeft = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Left); + Matrix4x4 stereoProjLeft = GL.GetGPUProjectionMatrix(cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left), true); + Matrix4x4 matVPLeft = stereoProjLeft * stereoViewLeft; + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.ViewProjMatrixLeft, matVPLeft); + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixMVLeft, stereoViewLeft * matO2W); + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixProjLeft, stereoProjLeft); + + Matrix4x4 stereoViewRight = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Right); + Matrix4x4 stereoProjRight = GL.GetGPUProjectionMatrix(cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right), true); + Matrix4x4 matVPRight = stereoProjRight * stereoViewRight; + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.ViewProjMatrixRight, matVPRight); + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixMVRight, stereoViewRight * matO2W); + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixProjRight, stereoProjRight); + + cmb.SetComputeIntParam(m_CSSplatUtilities, Props.IsStereo, 1); + } + else + { + cmb.SetComputeIntParam(m_CSSplatUtilities, Props.IsStereo, 0); + } cmb.SetComputeVectorParam(m_CSSplatUtilities, Props.VecScreenParams, screenPar); cmb.SetComputeVectorParam(m_CSSplatUtilities, Props.VecWorldSpaceCameraPos, camPos); @@ -606,7 +715,7 @@ internal void CalcViewData(CommandBuffer cmb, Camera cam) cmb.SetComputeIntParam(m_CSSplatUtilities, Props.SHOnly, m_SHOnly ? 1 : 0); m_CSSplatUtilities.GetKernelThreadGroupSizes((int)KernelIndices.CalcViewData, out uint gsX, out _, out _); - cmb.DispatchCompute(m_CSSplatUtilities, (int)KernelIndices.CalcViewData, (m_GpuView.count + (int)gsX - 1)/(int)gsX, 1, 1); + cmb.DispatchCompute(m_CSSplatUtilities, (int)KernelIndices.CalcViewData, (m_SplatCount + (int)gsX - 1)/(int)gsX, 1, 1); } internal void SortPoints(CommandBuffer cmd, Camera cam, Matrix4x4 matrix) @@ -634,7 +743,8 @@ internal void SortPoints(CommandBuffer cmd, Camera cam, Matrix4x4 matrix) // sort the splats EnsureSorterAndRegister(); - m_Sorter.Dispatch(cmd, m_SorterArgs); + if (m_Sorter.Valid) + m_Sorter.Dispatch(cmd, m_SorterArgs); cmd.EndSample(s_ProfSort); } @@ -997,7 +1107,8 @@ public void EditSetSplatCount(int newSplatCount) ClearGraphicsBuffer(newEditSelectedMouseDown); ClearGraphicsBuffer(newEditDeleted); - var newGpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, newSplatCount, kGpuViewDataSize); + // Double the size to hold both left and right eye data + var newGpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, newSplatCount * 2, kGpuViewDataSize); InitSortBuffers(newSplatCount); // copy existing data over into new buffers diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index cab17475..2843aa23 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -10,6 +10,7 @@ using UnityEngine.Rendering; using UnityEngine.Rendering.Universal; using UnityEngine.Rendering.RenderGraphModule; +using UnityEngine.XR; namespace GaussianSplatting.Runtime { @@ -34,6 +35,7 @@ class PassData internal TextureHandle SourceTexture; internal TextureHandle SourceDepth; internal TextureHandle GaussianSplatRT; + internal bool IsStereo; } public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer frameData) @@ -43,31 +45,83 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer var cameraData = frameData.Get(); var resourceData = frameData.Get(); + // isStereo requires the actual render target to be a Tex2DArray (main XR swapchain). + // OVROverlayCanvas and other stereo-enabled-but-2D cameras must take the non-stereo path. + bool isStereo = XRSettings.enabled && cameraData.camera.stereoEnabled && + (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview) && + !Application.isEditor && + cameraData.cameraTargetDescriptor.dimension == TextureDimension.Tex2DArray; + // Always use cameraTargetDescriptor — it matches the actual depth buffer size (including render scale). + // XRSettings.eyeTextureDesc returns the unscaled XR eye texture and causes dimension mismatches. RenderTextureDescriptor rtDesc = cameraData.cameraTargetDescriptor; rtDesc.depthBufferBits = 0; rtDesc.msaaSamples = 1; rtDesc.graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - var textureHandle = UniversalRenderer.CreateRenderGraphTexture(renderGraph, rtDesc, GaussianSplatRTName, true); + + // Create render texture + var gaussianSplatRT = UniversalRenderer.CreateRenderGraphTexture(renderGraph, rtDesc, GaussianSplatRTName, true); passData.CameraData = cameraData; passData.SourceTexture = resourceData.activeColorTexture; passData.SourceDepth = resourceData.activeDepthTexture; - passData.GaussianSplatRT = textureHandle; + passData.GaussianSplatRT = gaussianSplatRT; + passData.IsStereo = isStereo; builder.UseTexture(resourceData.activeColorTexture, AccessFlags.ReadWrite); builder.UseTexture(resourceData.activeDepthTexture); - builder.UseTexture(textureHandle, AccessFlags.Write); + builder.UseTexture(gaussianSplatRT, AccessFlags.ReadWrite); builder.AllowPassCulling(false); builder.SetRenderFunc(static (PassData data, UnsafeGraphContext context) => { var commandBuffer = CommandBufferHelpers.GetNativeCommandBuffer(context.cmd); using var _ = new ProfilingScope(commandBuffer, s_profilingSampler); - commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); - CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, data.SourceDepth, ClearFlag.Color, Color.clear); - Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer); - commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); - Blitter.BlitCameraTexture(commandBuffer, data.GaussianSplatRT, data.SourceTexture, matComposite, 0); - commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + + if (data.IsStereo) + { + // Sort once, render twice — sorting uses center eye matrix so per-eye sort + // produces identical results at 2x the GPU cost. + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear); + + var renderData = GaussianSplatRenderSystem.instance.PrepareSplats(data.CameraData.camera, commandBuffer); + + // [Quest3] Workaround: Unity doesn't correctly set unity_stereoEyeIndex when drawing to + // a render texture array, so we draw each eye manually. + + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.None, Color.clear, 0, CubemapFace.Unknown, 0); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 0); + + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.None, Color.clear, 0, CubemapFace.Unknown, 1); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 1); + Material matComposite = renderData.matComposite; + + // Composite to the final target + commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); + matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); + + // [Quest3] Workaround for stereo rendering. Unity is not able to correctly set unity_stereoEyeIndex when drawing to + // a render texture array, so we need to do it manually. Also, we need to draw the same material twice, + // once for each eye. TODO: Revisit this when Unity fixes the issue. + commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); + commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 0); // emulate left + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + + commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 1); + commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 1); // emulate right + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + } + else + { + // Single-eye rendering + commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, data.SourceDepth, ClearFlag.Color, Color.clear); + Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer); + + commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); + Blitter.BlitCameraTexture(commandBuffer, data.GaussianSplatRT, data.SourceTexture, matComposite, 0); + commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + } }); } } diff --git a/package/Shaders/GaussianComposite.shader b/package/Shaders/GaussianComposite.shader index 7a953ab7..72b136eb 100644 --- a/package/Shaders/GaussianComposite.shader +++ b/package/Shaders/GaussianComposite.shader @@ -15,6 +15,11 @@ CGPROGRAM #pragma fragment frag #pragma require compute #pragma use_dxc +#pragma require 2darray + +// Enable proper multi-compile support for all stereo rendering modes +#pragma multi_compile_local _ UNITY_SINGLE_PASS_STEREO STEREO_INSTANCING_ON STEREO_MULTIVIEW_ON + #include "UnityCG.cginc" struct v2f @@ -22,20 +27,40 @@ struct v2f float4 vertex : SV_POSITION; }; +struct appdata +{ + float4 vertex : POSITION; + uint vtxID : SV_VertexID; +}; + v2f vert (uint vtxID : SV_VertexID) { v2f o; + float2 quadPos = float2(vtxID&1, (vtxID>>1)&1) * 4.0 - 1.0; - o.vertex = float4(quadPos, 1, 1); + o.vertex = float4(quadPos, 1, 1); return o; } +// Separate textures for left and right eyes +#if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) +UNITY_DECLARE_TEX2DARRAY(_GaussianSplatRT); +#else Texture2D _GaussianSplatRT; +#endif +int _CustomStereoEyeIndex; half4 frag (v2f i) : SV_Target { - half4 col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); - return float4(GammaToLinearSpace(col.rgb/col.a),col.a); + half4 col; + #if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) + float2 normalizedUV = float2(i.vertex.x / _ScreenParams.x, i.vertex.y / _ScreenParams.y); + col = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, _CustomStereoEyeIndex)); + #else + col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); + #endif + + return float4(GammaToLinearSpace(col.rgb / col.a), col.a); } ENDCG } diff --git a/package/Shaders/RenderGaussianSplats.shader b/package/Shaders/RenderGaussianSplats.shader index 540a9f5b..f8409d67 100644 --- a/package/Shaders/RenderGaussianSplats.shader +++ b/package/Shaders/RenderGaussianSplats.shader @@ -31,12 +31,15 @@ struct v2f StructuredBuffer _SplatViewData; ByteAddressBuffer _SplatSelectedBits; uint _SplatBitsValid; - +uint _EyeIndex; +uint _IsStereo; v2f vert (uint vtxID : SV_VertexID, uint instID : SV_InstanceID) { - v2f o = (v2f)0; - instID = _OrderBuffer[instID]; - SplatViewData view = _SplatViewData[instID]; + v2f o = (v2f)0; + instID = _OrderBuffer[instID]; + uint eyeIndex = _EyeIndex; + uint viewIndex = _IsStereo ? instID * 2 + eyeIndex : instID; + SplatViewData view = _SplatViewData[viewIndex]; float4 centerClipPos = view.pos; bool behindCam = centerClipPos.w <= 0; if (behindCam) @@ -100,8 +103,7 @@ half4 frag (v2f i) : SV_Target i.col.rgb = lerp(i.col.rgb, selectedColor, 0.5); } - if (alpha < 1.0/255.0) - discard; + clip(alpha - 1.0/255.0); half4 res = half4(i.col.rgb * alpha, alpha); return res; diff --git a/package/Shaders/SplatUtilities.compute b/package/Shaders/SplatUtilities.compute index 77241dae..3df3dae7 100644 --- a/package/Shaders/SplatUtilities.compute +++ b/package/Shaders/SplatUtilities.compute @@ -81,13 +81,23 @@ void CSCalcDistances (uint3 id : SV_DispatchThreadID) _SplatSortDistances[idx] = FloatToSortableUint(pos.z); } +cbuffer StereoMatrices +{ + float4x4 _ViewProjMatrixLeft; + float4x4 _ViewProjMatrixRight; + float4x4 _MatrixMVLeft; + float4x4 _MatrixMVRight; + float4x4 _MatrixProjLeft; + float4x4 _MatrixProjRight; +}; + RWStructuredBuffer _SplatViewData; float _SplatScale; float _SplatOpacityScale; uint _SHOrder; uint _SHOnly; - +uint _IsStereo; uint _SplatCutoutsCount; #define SPLAT_CUTOUT_TYPE_ELLIPSOID 0 @@ -106,47 +116,6 @@ uint _SplatBitsValid; void DecomposeCovariance(float3 cov2d, out float2 v1, out float2 v2) { - #if 0 // does not quite give the correct results? - - // https://jsfiddle.net/mattrossman/ehxmtgw6/ - // References: - // - https://www.youtube.com/watch?v=e50Bj7jn9IQ - // - https://en.wikipedia.org/wiki/Eigenvalue_algorithm#2%C3%972_matrices - // - https://people.math.harvard.edu/~knill/teaching/math21b2004/exhibits/2dmatrices/index.html - float a = cov2d.x; - float b = cov2d.y; - float d = cov2d.z; - float det = a * d - b * b; // matrix is symmetric, so "c" is same as "b" - float trace = a + d; - - float mean = 0.5 * trace; - float dist = sqrt(mean * mean - det); - - float lambda1 = mean + dist; // 1st eigenvalue - float lambda2 = mean - dist; // 2nd eigenvalue - - if (b == 0) { - // https://twitter.com/the_ross_man/status/1706342719776551360 - if (a > d) v1 = float2(1, 0); - else v1 = float2(0, 1); - } else - v1 = normalize(float2(b, d - lambda2)); - - v1.y = -v1.y; - // The 2nd eigenvector is just a 90 degree rotation of the first since Gaussian axes are orthogonal - v2 = float2(v1.y, -v1.x); - - // scaling components - v1 *= sqrt(lambda1); - v2 *= sqrt(lambda2); - - float radius = 1.5; - v1 *= radius; - v2 *= radius; - - #else - - // same as in antimatter15/splat float diag1 = cov2d.x, diag2 = cov2d.z, offDiag = cov2d.y; float mid = 0.5f * (diag1 + diag2); float radius = length(float2((diag1 - diag2) / 2.0, offDiag)); @@ -157,8 +126,6 @@ void DecomposeCovariance(float3 cov2d, out float2 v1, out float2 v2) float maxSize = 4096.0; v1 = min(sqrt(2.0 * lambda1), maxSize) * diagVec; v2 = min(sqrt(2.0 * lambda2), maxSize) * float2(diagVec.y, -diagVec.x); - - #endif } bool IsSplatCut(float3 pos) @@ -186,60 +153,35 @@ bool IsSplatCut(float3 pos) return finalCut; } -[numthreads(GROUP_SIZE,1,1)] -void CSCalcViewData (uint3 id : SV_DispatchThreadID) +SplatViewData CalculateEyeViewData(SplatData splat, float3 centerWorldPos, float4x4 viewProjMatrix, + float4x4 matMV, float4x4 matProj, float4 screenParams, + bool isDeleted, bool isCut, float splatScale, half opacityScale) { - uint idx = id.x; - if (idx >= _SplatCount) - return; - - SplatData splat = LoadSplatData(idx); SplatViewData view = (SplatViewData)0; - float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz; - float4 centerClipPos = mul(UNITY_MATRIX_VP, float4(centerWorldPos, 1)); - half opacityScale = _SplatOpacityScale; - float splatScale = _SplatScale; - - // deleted? - if (_SplatBitsValid) - { - uint wordIdx = idx / 32; - uint bitIdx = idx & 31; - uint wordVal = _SplatDeletedBits.Load(wordIdx * 4); - if (wordVal & (1 << bitIdx)) - { - centerClipPos.w = 0; - } - } - - // cutouts - if (IsSplatCut(splat.pos)) + // Calculate projection + float4 centerClipPos = mul(viewProjMatrix, float4(centerWorldPos, 1)); + if (isDeleted || isCut) { centerClipPos.w = 0; } - view.pos = centerClipPos; + bool behindCam = centerClipPos.w <= 0; if (!behindCam) { - float4 boxRot = splat.rot; - float3 boxSize = splat.scale; - - float3x3 splatRotScaleMat = CalcMatrixFromRotationScale(boxRot, boxSize); - + float3x3 splatRotScaleMat = CalcMatrixFromRotationScale(splat.rot, splat.scale); float3 cov3d0, cov3d1; CalcCovariance3D(splatRotScaleMat, cov3d0, cov3d1); float splatScale2 = splatScale * splatScale; cov3d0 *= splatScale2; cov3d1 *= splatScale2; - float3 cov2d = CalcCovariance2D(splat.pos, cov3d0, cov3d1, _MatrixMV, UNITY_MATRIX_P, _VecScreenParams); - + float3 cov2d = CalcCovariance2D(splat.pos, cov3d0, cov3d1, matMV, matProj, screenParams); + DecomposeCovariance(cov2d, view.axis1, view.axis2); float3 worldViewDir = _VecWorldSpaceCameraPos.xyz - centerWorldPos; - float3 objViewDir = mul((float3x3)_MatrixWorldToObject, worldViewDir); - objViewDir = normalize(objViewDir); + float3 objViewDir = normalize(mul((float3x3)_MatrixWorldToObject, worldViewDir)); half4 col; col.rgb = ShadeSH(splat.sh, objViewDir, _SHOrder, _SHOnly != 0); @@ -248,7 +190,63 @@ void CSCalcViewData (uint3 id : SV_DispatchThreadID) view.color.y = (f32tof16(col.b) << 16) | f32tof16(col.a); } - _SplatViewData[idx] = view; + return view; +} + +[numthreads(GROUP_SIZE,1,1)] +void CSCalcViewData (uint3 id : SV_DispatchThreadID) +{ + uint idx = id.x; + if (idx >= _SplatCount) + return; + + SplatData splat = LoadSplatData(idx); + + // Transform to world space + float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz; + float splatScale = _SplatScale; + half opacityScale = _SplatOpacityScale; + + // Check if deleted + bool isDeleted = false; + if (_SplatBitsValid) + { + uint wordIdx = idx / 32; + uint bitIdx = idx & 31; + uint wordVal = _SplatDeletedBits.Load(wordIdx * 4); + if (wordVal & (1 << bitIdx)) + { + isDeleted = true; + } + } + + // Check if cut + bool isCut = IsSplatCut(splat.pos); + + if (_IsStereo) + { + // Compute full covariance, SH, and color using the left eye (expensive). + // Reuse axes and color for the right eye — the IPD offset has negligible + // effect on projected 2D shape. Only the clip position differs per eye. + SplatViewData viewLeft = CalculateEyeViewData(splat, centerWorldPos, _ViewProjMatrixLeft, + _MatrixMVLeft, _MatrixProjLeft, _VecScreenParams, + isDeleted, isCut, splatScale, opacityScale); + + SplatViewData viewRight = viewLeft; + float4 clipRight = mul(_ViewProjMatrixRight, float4(centerWorldPos, 1)); + if (isDeleted || isCut) + clipRight.w = 0; + viewRight.pos = clipRight; + + _SplatViewData[idx * 2] = viewLeft; + _SplatViewData[idx * 2 + 1] = viewRight; + } + else + { + _SplatViewData[idx] = CalculateEyeViewData(splat, centerWorldPos, UNITY_MATRIX_VP, + _MatrixMV, UNITY_MATRIX_P, _VecScreenParams, + isDeleted, isCut, splatScale, opacityScale); + } }