actually loosened tolerances (#47)

knightron0 · web-flow · commit bea919930e77 · 2026-01-20T02:56:43.000-05:00
* principled tol

* switch dtype stuff

* update tolerances

* edit
diff --git a/problems/avg-pool-1d/def.py b/problems/avg-pool-1d/def.py
@@ -110,7 +110,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=4e-4, atol=2e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/avg-pool-2d/def.py b/problems/avg-pool-2d/def.py
@@ -108,7 +108,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=4e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/avg-pool-3d/def.py b/problems/avg-pool-3d/def.py
@@ -109,7 +109,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=5e-4, atol=8e-5)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/batch-norm/def.py b/problems/batch-norm/def.py
@@ -104,11 +104,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        # Use a slightly higher tolerance for BatchNorm due to potential precision differences
-        rtol = 1e-3 if dtype == torch.float16 else 1e-4
-        atol = 1e-3 if dtype == torch.float16 else 1e-5
-        
-        is_close = torch.allclose(actual_output, expected_output, rtol=rtol, atol=atol)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=1e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/conv-1d/def.py b/problems/conv-1d/def.py
@@ -108,7 +108,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=3e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/conv-2d/def.py b/problems/conv-2d/def.py
@@ -115,7 +115,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=4e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/conv-square-3d/def.py b/problems/conv-square-3d/def.py
@@ -103,7 +103,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/conv2d-relu-hardswish/def.py b/problems/conv2d-relu-hardswish/def.py
@@ -120,7 +120,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=9e-4, atol=5e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/cosine-similarity/def.py b/problems/cosine-similarity/def.py
@@ -99,7 +99,7 @@ def verify_result(self, expected_output: torch.Tensor,
                  "message": "Output shape mismatch, expected: {}, actual: {}".format(expected_output.shape, actual_output.shape),
              }
 
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=2e-4, atol=2e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/cumprod/def.py b/problems/cumprod/def.py
@@ -87,7 +87,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-2)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-2, atol=2e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/cumsum/def.py b/problems/cumsum/def.py
@@ -85,7 +85,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-2)
+        is_close = torch.allclose(actual_output, expected_output, rtol=3e-2, atol=1e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/diagonal-matmul/def.py b/problems/diagonal-matmul/def.py
@@ -99,7 +99,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=5e-4, atol=3e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/elu/def.py b/problems/elu/def.py
@@ -94,7 +94,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=2e-4, atol=2e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/frobenius-norm/def.py b/problems/frobenius-norm/def.py
@@ -112,7 +112,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """        
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=3e-3, atol=1e-6)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/gelu/def.py b/problems/gelu/def.py
@@ -88,7 +88,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=4e-4, atol=2e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/gemm-multiply-leakyrelu/def.py b/problems/gemm-multiply-leakyrelu/def.py
@@ -125,7 +125,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/gemm-relu/def.py b/problems/gemm-relu/def.py
@@ -115,7 +115,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=7e-4, atol=3e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/hard-sigmoid/def.py b/problems/hard-sigmoid/def.py
@@ -94,7 +94,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=2e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/hinge-loss/def.py b/problems/hinge-loss/def.py
@@ -89,7 +89,7 @@ def verify_result(self, expected_output: torch.Tensor,
             Tuple of (is_correct, debug_info)
         """
 
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=4e-4, atol=1e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/huber-loss/def.py b/problems/huber-loss/def.py
@@ -95,7 +95,7 @@ def verify_result(self, expected_output: torch.Tensor,
                  "message": "Output shape mismatch, expected: {}, actual: {}".format(expected_output.shape, actual_output.shape),
              }
 
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=8e-4, atol=1e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/l1-norm/def.py b/problems/l1-norm/def.py
@@ -102,11 +102,8 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        # Use appropriate tolerance based on dtype
-        rtol = 1e-3 if dtype == torch.float16 else 1e-4
-        atol = 1e-3 if dtype == torch.float16 else 1e-5
-        
-        is_close = torch.allclose(actual_output, expected_output, rtol=rtol, atol=atol)
+
+        is_close = torch.allclose(actual_output, expected_output, rtol=7e-4, atol=5e-5)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/l2-norm/def.py b/problems/l2-norm/def.py
@@ -102,11 +102,8 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        # Use appropriate tolerance based on dtype
-        rtol = 1e-3 if dtype == torch.float16 else 1e-4
-        atol = 1e-3 if dtype == torch.float16 else 1e-5
         
-        is_close = torch.allclose(actual_output, expected_output, rtol=rtol, atol=atol)
+        is_close = torch.allclose(actual_output, expected_output, rtol=7e-4, atol=1e-5)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/layer-norm/def.py b/problems/layer-norm/def.py
@@ -121,11 +121,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        # Use a slightly higher tolerance for LayerNorm due to potential precision differences
-        rtol = 1e-3 if dtype == torch.float16 else 1e-4
-        atol = 1e-3 if dtype == torch.float16 else 1e-5
-        
-        is_close = torch.allclose(actual_output, expected_output, rtol=rtol, atol=atol)
+        is_close = torch.allclose(actual_output, expected_output, rtol=9e-4, atol=2e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/leaky-relu/def.py b/problems/leaky-relu/def.py
@@ -99,7 +99,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/lower-trig-matmul/def.py b/problems/lower-trig-matmul/def.py
@@ -111,7 +111,7 @@ def verify_result(self, expected_output: torch.Tensor,
         # Ensure the actual output is also lower triangular (optional check)
         is_lower_triangular = torch.all(torch.triu(actual_output, diagonal=1) == 0)
         
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-3) # Relaxed tolerance slightly
+        is_close = torch.allclose(actual_output, expected_output, rtol=8e-4, atol=2e-2) # Relaxed tolerance slightly
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matmul-3d/def.py b/problems/matmul-3d/def.py
@@ -101,7 +101,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=3e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matmul-4d/def.py b/problems/matmul-4d/def.py
@@ -101,7 +101,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=5e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matmul-sigmoid-sum/def.py b/problems/matmul-sigmoid-sum/def.py
@@ -111,7 +111,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=5e-2, atol=1e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matmul-swish-scaling/def.py b/problems/matmul-swish-scaling/def.py
@@ -124,7 +124,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=8e-4, atol=1e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matmul-swish/def.py b/problems/matmul-swish/def.py
@@ -109,7 +109,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=2e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matrix-multiplication/def.py b/problems/matrix-multiplication/def.py
@@ -119,7 +119,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=3e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matrix-scalar/def.py b/problems/matrix-scalar/def.py
@@ -86,7 +86,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=3e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/matrix-vector/def.py b/problems/matrix-vector/def.py
@@ -92,7 +92,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=6e-4, atol=2e-2)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/max-pool-1d/def.py b/problems/max-pool-1d/def.py
@@ -115,7 +115,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/max-pool-2d/def.py b/problems/max-pool-2d/def.py
@@ -115,7 +115,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/max-pool-3d/def.py b/problems/max-pool-3d/def.py
@@ -118,7 +118,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/mean-dim/def.py b/problems/mean-dim/def.py
@@ -94,7 +94,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=5e-4, atol=7e-5)
 
         debug_info = {}
         if not is_close:
diff --git a/problems/mse-loss/def.py b/problems/mse-loss/def.py
@@ -87,7 +87,7 @@ def verify_result(self, expected_output: torch.Tensor,
             Tuple of (is_correct, debug_info)
         """
         # Check if outputs are close (should be scalar values)
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=3e-4)
                 
         debug_info = {}
         if not is_close:
diff --git a/problems/product-dim/def.py b/problems/product-dim/def.py
@@ -55,7 +55,7 @@ def generate_test_cases(self, dtype: torch.dtype) -> List[Dict[str, Any]]:
                 "dim": dim,
                 "create_inputs": lambda shape=shape, dim=dim, seed=seed, dtype=dtype: (
                     *(lambda g: (
-                        torch.rand(shape, device="cuda", dtype=dtype, generator=g) * 1.0 - 0.5,  # uniform [-0.5, 0.5]
+                        torch.rand(shape, device="cuda", dtype=dtype, generator=g) * 4.0 - 2.0,  # uniform [-2.0, 2.0]
                     ))(torch.Generator(device="cuda").manual_seed(seed)),
                     dim
                 )
@@ -95,7 +95,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-3, atol=1e-3)
+        is_close = torch.allclose(actual_output, expected_output, rtol=2e-2, atol=7e-3)
 
         debug_info = {}
         if not is_close:
diff --git a/problems/relu/def.py b/problems/relu/def.py
@@ -96,7 +96,7 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)
+        is_close = torch.allclose(actual_output, expected_output, rtol=1e-4, atol=1e-4)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/rms-norm/def.py b/problems/rms-norm/def.py
@@ -95,11 +95,8 @@ def verify_result(self, expected_output: torch.Tensor,
         Returns:
             Tuple of (is_correct, debug_info)
         """
-        # Use appropriate tolerance based on precision
-        rtol = 1e-3 if dtype == torch.float16 else 1e-4
-        atol = 1e-3 if dtype == torch.float16 else 1e-5
         
-        is_close = torch.allclose(actual_output, expected_output, rtol=rtol, atol=atol)
+        is_close = torch.allclose(actual_output, expected_output, rtol=8e-4, atol=1e-3)
         
         debug_info = {}
         if not is_close:
diff --git a/problems/running-sum-1d/def.py b/problems/running-sum-1d/def.py
diff --git a/problems/scaled-dot-attention/def.py b/problems/scaled-dot-attention/def.py
diff --git a/problems/selu/def.py b/problems/selu/def.py
diff --git a/problems/sigmoid/def.py b/problems/sigmoid/def.py
diff --git a/problems/soft-plus/def.py b/problems/soft-plus/def.py
diff --git a/problems/softmax/def.py b/problems/softmax/def.py
diff --git a/problems/square-matmul/def.py b/problems/square-matmul/def.py
diff --git a/problems/sum-dim/def.py b/problems/sum-dim/def.py
diff --git a/problems/swish/def.py b/problems/swish/def.py
diff --git a/problems/symmetric-matmul/def.py b/problems/symmetric-matmul/def.py
diff --git a/problems/tanh/def.py b/problems/tanh/def.py
diff --git a/problems/triplet-margin/def.py b/problems/triplet-margin/def.py
diff --git a/problems/upper-trig-matmul/def.py b/problems/upper-trig-matmul/def.py
diff --git a/problems/vector-addition/def.py b/problems/vector-addition/def.py

Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ def verify_result(self, expected_output: torch.Tensor,`
`99`	`99`	`"message": "Output shape mismatch, expected: {}, actual: {}".format(expected_output.shape, actual_output.shape),`
`100`	`100`	`}`
`101`	`101`
`102`		`- is_close = torch.allclose(actual_output, expected_output, rtol=1e-5, atol=1e-5)`
	`102`	`+ is_close = torch.allclose(actual_output, expected_output, rtol=2e-4, atol=2e-4)`
`103`	`103`
`104`	`104`	`debug_info = {}`
`105`	`105`	`if not is_close:`