Refactored DescriptorSets

Made descriptor sets way more more performant
Added DescriptorReflection to ShaderCookers output of ShaderPacks
Changed ShaderCooker to compile .slang files instead of .hlsl
Removed old DescriptorSetBuilder in favor of new DescriptorHandlerVK
Removed ModelDesc (stupidly old unused code)
Added Math functions for u32
Added a few tracy zones

Author: Pursche

Source/Base/Base/Math/Math.h

@@ -123,6 +123,16 @@ namespace Math
         return (a >= b) ? a : b;
     }
 
+    inline u32 Min(u32 a, u32 b)
+    {
+        return (a <= b) ? a : b;
+    }
+
+    inline u32 Max(u32 a, u32 b)
+    {
+        return (a >= b) ? a : b;
+    }
+
     inline size_t Min(size_t a, size_t b)
     {
         return (a <= b) ? a : b;
@@ -143,6 +153,11 @@ namespace Math
         return Max(Min(x, max), min);
     }
 
+    inline u32 Clamp(u32 x, u32 min, u32 max)
+    {
+        return Max(Min(x, max), min);
+    }
+
     inline size_t Clamp(size_t x, size_t min, size_t max)
     {
         return Max(Min(x, max), min);

Source/Base/Base/Memory/Bytebuffer.cpp

@@ -15,6 +15,7 @@ SharedPool<Bytebuffer> Bytebuffer::_byteBuffer524288;
 SharedPool<Bytebuffer> Bytebuffer::_byteBuffer1048576;
 SharedPool<Bytebuffer> Bytebuffer::_byteBuffer8388608;
 SharedPool<Bytebuffer> Bytebuffer::_byteBuffer16777216;
+SharedPool<Bytebuffer> Bytebuffer::_byteBuffer67108864;
 SharedPool<Bytebuffer> Bytebuffer::_byteBuffer209715200;
 
 std::shared_ptr<Bytebuffer> Bytebuffer::BorrowRuntime(size_t size)

Source/Base/Base/Memory/Bytebuffer.h

@@ -692,6 +692,14 @@ class Bytebuffer
 
             return buffer;
         }
+        else if constexpr (size <= 67108864)
+        {
+            std::shared_ptr<Bytebuffer> buffer = _byteBuffer67108864.acquireOrCreate(nullptr, 67108864);
+            buffer->size = size;
+            buffer->Reset();
+
+            return buffer;
+        }
         else if constexpr (size <= 209715200) // This is used for the Data Extractor, largest observed file in WOTLK is 65MB, however in BFA this has been observed to be 150MB
         {
             std::shared_ptr<Bytebuffer> buffer = _byteBuffer209715200.acquireOrCreate(nullptr, 209715200);
@@ -721,7 +729,8 @@ class Bytebuffer
     static SharedPool<Bytebuffer> _byteBuffer524288;
     static SharedPool<Bytebuffer> _byteBuffer1048576;
     static SharedPool<Bytebuffer> _byteBuffer8388608;
-    static SharedPool<Bytebuffer> _byteBuffer16777216;
+    static SharedPool<Bytebuffer> _byteBuffer16777216; 
+    static SharedPool<Bytebuffer> _byteBuffer67108864;
     static SharedPool<Bytebuffer> _byteBuffer209715200;
 
     class RuntimePool

Source/Base/Base/Platform.h

@@ -68,5 +68,11 @@ static_assert(false, "Please add PRAGMA_ENABLE_OPTIMIZATION/PRAGMA_DISABLE_OPTIM
 #ifndef strcpy_s
 #define strcpy_s(dest, count)  strcpy((dest), (count))
 #endif // strcpy_s
-#define ARRAYSIZE(arr) (sizeof(arr)/sizeof(arr[0]))
 #endif
+
+#ifndef ARRAY_COUNT
+#define ARRAY_COUNT(arr) (sizeof(arr)/sizeof(arr[0]))
+#endif
+
+#define STATIC_ASSERT_MATCHING_ARRAY(arrA, arrB) static_assert(ARRAY_COUNT(arrA) == ARRAY_COUNT(arrB), "Array count mismatch: " #arrA " and " #arrB)
+#define STATIC_ASSERT_MATCHING_ENUM_ARRAY(enumType, arr) static_assert((static_cast<u32>(enumType::COUNT) + 1) == ARRAY_COUNT(arr), "Enum and array count mismatch: " #enumType " and " #arr)

Source/FileFormat/FileFormat/Novus/ShaderPack/ShaderPack.cpp

@@ -7,6 +7,31 @@
 
 namespace FileFormat
 {
+    std::string DescriptorReflection::ToString() const
+    {
+        std::string result = "DescriptorReflection: { ";
+        result += "Binding: " + std::to_string(binding) + ", ";
+        result += "Name: " + name + ", ";
+        result += "Type: " + std::string(DescriptorTypeReflectionNames[static_cast<u32>(type)]) + ", ";
+        result += "SubType: " + std::string(DescriptorTypeReflectionNames[static_cast<u32>(subType)]) + ", ";
+        result += "Count: " + std::to_string(count);
+        result += " }";
+        return result;
+    }
+
+    DescriptorSetReflection& ShaderReflection::GetDescriptorSetBySlot(u32 slot)
+    {
+        if (!descriptorSets.contains(slot))
+        {
+            DescriptorSetReflection descriptorSetReflection;
+            descriptorSetReflection.slot = slot;
+
+            descriptorSets[slot] = descriptorSetReflection;
+        }
+
+        return descriptorSets[slot];
+    }
+
     ShaderRef* ShaderPack::GetShaderRef(std::shared_ptr<Bytebuffer>& buffer, u32 shaderIndex)
     {
         if (shaderIndex >= manifest.numShaders)
@@ -18,6 +43,79 @@ namespace FileFormat
         return shaderRef;
     }
 
+    bool ShaderPack::GetShaderReflection(std::shared_ptr<Bytebuffer>& buffer, u32 shaderIndex, ShaderReflection& reflection)
+    {
+        ShaderRef* shaderRef = GetShaderRef(buffer, shaderIndex);
+        if (shaderRef == nullptr || shaderRef->reflectionOffset == 0 || shaderRef->reflectionSize == 0)
+            return false;
+
+        u64 readOffset = buffer->readData;
+
+        buffer->readData = shaderRef->reflectionOffset;
+        u8 numDescriptorSets;
+        if (!buffer->GetU8(numDescriptorSets))
+            return false;
+
+        reflection.descriptorSets.reserve(numDescriptorSets);
+
+        for(u32 i = 0; i < numDescriptorSets; i++)
+        {
+            u8 setSlot;
+            u8 numDescriptors;
+
+            if (!buffer->GetU8(setSlot))
+                return false;
+
+            if (!buffer->GetU8(numDescriptors))
+                return false;
+
+            DescriptorSetReflection& descriptorSet = reflection.descriptorSets[setSlot];
+            descriptorSet.slot = setSlot;
+            descriptorSet.descriptors.reserve(numDescriptors);
+
+            for (u32 j = 0; j < numDescriptors; j++)
+            {
+                u8 binding;
+                std::string name;
+                u8 type;
+                u8 subType;
+                u16 count;
+                u8 accessType;
+                bool isUsed;
+                u32 byteOffset;
+                u32 byteSize;
+
+                bool failed = false;
+                failed |= !buffer->GetU8(binding);
+                failed |= !buffer->GetString(name);
+                failed |= !buffer->GetU8(type);
+                failed |= !buffer->GetU8(subType);
+                failed |= !buffer->GetU16(count);
+                failed |= !buffer->GetU8(accessType);
+                failed |= !buffer->Get(isUsed);
+                failed |= !buffer->GetU32(byteOffset);
+                failed |= !buffer->GetU32(byteSize);
+
+                if (failed)
+                    return false;
+
+                DescriptorReflection& descriptor = descriptorSet.descriptors[binding];
+                descriptor.binding = binding;
+                descriptor.name = std::move(name);
+                descriptor.type = static_cast<DescriptorTypeReflection>(type);
+                descriptor.subType = static_cast<DescriptorTypeReflection>(subType);
+                descriptor.count = count;
+                descriptor.accessType = static_cast<DescriptorAccessTypeReflection>(accessType);
+                descriptor.isUsed = isUsed;
+                descriptor.byteOffset = byteOffset;
+                descriptor.byteSize = byteSize;
+            }
+        }
+
+        buffer->readData = readOffset;
+        return true;
+    }
+
     u8* ShaderPack::GetShaderDataPtr(std::shared_ptr<Bytebuffer>& buffer, u32 shaderOffset)
     {
         if (shaderOffset >= buffer->writtenData)
@@ -39,29 +137,74 @@ namespace FileFormat
         manifest.numShaders = static_cast<u32>(shaders.size());
         NC_ASSERT(shaders.size() > 0, "ShaderPack must have at least one shader");
 
-        // Write the Header and manifest to file
-        output.write(reinterpret_cast<char const*>(&header), sizeof(header));
-        output.write(reinterpret_cast<char const*>(&manifest), sizeof(manifest));
+        std::shared_ptr<Bytebuffer> buffer = Bytebuffer::Borrow<67108864>();
+
+        // Write the Header and manifest to buffer
+        buffer->Put(header);
+        buffer->Put(manifest);
 
         // Write the ShaderRefs to file
-        u64 dataOffset = static_cast<u64>(output.tellp()) + (sizeof(ShaderRef) * manifest.numShaders);
+        u64 baseShaderRefOffset = buffer->writtenData;
+        u64 dataOffset = baseShaderRefOffset + (sizeof(ShaderRef) * manifest.numShaders);
+        
         ShaderRef shaderRef;
         for (u32 i = 0; i < shaders.size(); i++)
         {
             shaderRef.permutationNameHash = shaders[i].permutationNameHash;
             shaderRef.dataOffset = dataOffset;
             shaderRef.dataSize = shaders[i].size;
 
-            output.write(reinterpret_cast<char const*>(&shaderRef), sizeof(ShaderRef));
+            buffer->Put(shaderRef);
             dataOffset += shaders[i].size;
         }
 
         // Write the shader blobs to file
         for (u32 i = 0; i < shaders.size(); i++)
         {
-            output.write(reinterpret_cast<char const*>(shaders[i].data), shaders[i].size);
+            buffer->PutBytes(shaders[i].data, shaders[i].size);
         }
 
+        // Write the reflection data to file
+        for (u32 i = 0; i < shaders.size(); i++)
+        {
+            u64 reflectionDataStartOffset = static_cast<u64>(buffer->writtenData);
+
+            const ShaderInMemory& shader = shaders[i];
+
+            // Write reflection data
+            u8 numDescriptorSets = static_cast<u8>(shader.reflection.descriptorSets.size());
+            buffer->PutU8(numDescriptorSets);
+            for(const auto& [setSlot, descriptorSet] : shader.reflection.descriptorSets)
+            {
+                buffer->PutU8(descriptorSet.slot);
+
+                u8 numDescriptors = static_cast<u8>(descriptorSet.descriptors.size());
+                buffer->PutU8(numDescriptors);
+                for (const auto& [binding, descriptor] : descriptorSet.descriptors)
+                {
+                    buffer->PutU8(descriptor.binding);
+                    buffer->PutString(descriptor.name);
+                    buffer->PutU8(static_cast<u8>(descriptor.type));
+                    buffer->PutU8(static_cast<u8>(descriptor.subType));
+                    buffer->PutU16(descriptor.count);
+                    buffer->PutU8(static_cast<u8>(descriptor.accessType));
+                    buffer->Put(descriptor.isUsed);
+                    buffer->PutU32(descriptor.byteOffset);
+                    buffer->PutU32(descriptor.byteSize);
+                }
+            }
+            
+            u64 reflectionDataEndOffset = static_cast<u64>(buffer->writtenData);
+
+            // Patch ShaderRef with reflection data offset and size
+            u64 shaderRefOffset = baseShaderRefOffset + (i * sizeof(ShaderRef));
+            ShaderRef* shaderRefInBuffer = reinterpret_cast<ShaderRef*>(buffer->GetDataPointer() + shaderRefOffset);
+            shaderRefInBuffer->reflectionOffset = reflectionDataStartOffset;
+            shaderRefInBuffer->reflectionSize = static_cast<u32>(reflectionDataEndOffset - reflectionDataStartOffset);
+        }
+
+        output.write(reinterpret_cast<char const*>(buffer->GetDataPointer()), buffer->writtenData);
+
         output.close();
 
         return true;