[DX-3716] Fixes Flaky VRFv2 Tests + CI Core Timeouts (#22324)

* Fixes flaky VRFv2 Tests

* Remove 90 second timeout

* go fix vrf/v2/...

* Linting

* More linting

* lint again

Author: kalverra

core/internal/testutils/testutils.go

@@ -86,17 +86,39 @@ func RandomizeName(n string) string {
 // DefaultWaitTimeout is the default wait timeout. If you have a *testing.T, use WaitTimeout instead.
 const DefaultWaitTimeout = 30 * time.Second
 
-// WaitTimeout returns a timeout based on the test's Deadline, if available.
+// deadlineRemainingBudget returns ~90% of time until the test deadline, or false if none.
+func deadlineRemainingBudget(t *testing.T) (time.Duration, bool) {
+	if d, ok := t.Deadline(); ok {
+		return time.Until(d) * 9 / 10, true // 10% buffer for cleanup
+	}
+	return 0, false
+}
+
+// WaitTimeout returns a timeout capped by the test's Deadline, if available.
 // Especially important to use in parallel tests, as their individual execution
 // can get paused for arbitrary amounts of time.
+//
+// When a deadline exists, it uses the full remaining budget (90% of time until the
+// deadline), not [DefaultWaitTimeout], so long-running tests still get enough wall
+// clock under package timeouts.
 func WaitTimeout(t *testing.T) time.Duration {
-	if d, ok := t.Deadline(); ok {
-		// 10% buffer for cleanup and scheduling delay
-		return time.Until(d) * 9 / 10
+	if budget, ok := deadlineRemainingBudget(t); ok {
+		return budget
 	}
 	return DefaultWaitTimeout
 }
 
+// WaitTimeoutCustom uses the requested duration when there is no test deadline.
+// When the test has a deadline, it returns the lesser of the requested duration and
+// the remaining budget (90% of time until deadline), so callers can ask for e.g. 5m
+// without exceeding the test process deadline.
+func WaitTimeoutCustom(t *testing.T, requested time.Duration) time.Duration {
+	if budget, ok := deadlineRemainingBudget(t); ok {
+		return min(budget, requested)
+	}
+	return requested
+}
+
 // Context returns a context with the test's deadline, if available.
 // Deprecated: use [testing.TB.Context] directly
 func Context(tb testing.TB) context.Context {

core/internal/testutils/testutils_test.go

@@ -0,0 +1,38 @@
+package testutils
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWaitTimeoutUsesDeadlineBudgetNotDefaultCap(t *testing.T) {
+	if d, ok := t.Deadline(); ok {
+		expectBefore := time.Until(d) * 9 / 10
+		got := WaitTimeout(t)
+		expectAfter := time.Until(d) * 9 / 10
+		require.Greater(t, got, time.Duration(0))
+		// got uses time.Until(deadline) inside WaitTimeout between these snapshots
+		require.GreaterOrEqual(t, got, expectAfter)
+		require.LessOrEqual(t, got, expectBefore)
+	} else {
+		require.Equal(t, DefaultWaitTimeout, WaitTimeout(t))
+	}
+}
+
+func TestWaitTimeoutCustom(t *testing.T) {
+	requested := 10 * time.Second
+
+	if d, ok := t.Deadline(); ok {
+		expectBefore := time.Until(d) * 9 / 10
+		got := WaitTimeoutCustom(t, requested)
+		expectAfter := time.Until(d) * 9 / 10
+		require.Greater(t, got, time.Duration(0))
+		require.LessOrEqual(t, got, requested)
+		require.GreaterOrEqual(t, got, min(expectAfter, requested))
+		require.LessOrEqual(t, got, min(expectBefore, requested))
+	} else {
+		require.Equal(t, requested, WaitTimeoutCustom(t, requested))
+	}
+}

core/services/vrf/v2/bhs_feeder_test.go

@@ -1,6 +1,7 @@
 package v2_test
 
 import (
+	"math/big"
 	"testing"
 	"time"
 
@@ -11,7 +12,6 @@ import (
 
 	"github.com/smartcontractkit/chainlink-evm/pkg/assets"
 	"github.com/smartcontractkit/chainlink-evm/pkg/config/toml"
-	"github.com/smartcontractkit/chainlink-evm/pkg/types"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/cltest"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
@@ -27,38 +27,38 @@ func TestStartHeartbeats(t *testing.T) {
 	vrfKey := cltest.MustGenerateRandomKey(t)
 	sendEth(t, ownerKey, uni.backend, vrfKey.Address, 10)
 	gasLanePriceWei := assets.GWei(1)
-	gasLimit := 3_000_000
+	gasLimit := uint64(3_000_000)
 
 	consumers := uni.vrfConsumers
 
 	// generate n BHS keys to make sure BHS job rotates sending keys
-	var bhsKeyAddresses []string
-	var keySpecificOverrides []toml.KeySpecific
-	var keys []any
+	bhsKeyAddresses := make([]string, 0, len(consumers))
+	keySpecificOverrides := make([]toml.KeySpecific, 0, len(consumers)+1)
+	keys := make([]any, 0, len(consumers)+2)
 	for range consumers {
 		bhsKey := cltest.MustGenerateRandomKey(t)
 		bhsKeyAddresses = append(bhsKeyAddresses, bhsKey.Address.String())
 		keys = append(keys, bhsKey)
 		keySpecificOverrides = append(keySpecificOverrides, toml.KeySpecific{
-			Key:          ptr[types.EIP55Address](bhsKey.EIP55Address),
+			Key:          new(bhsKey.EIP55Address),
 			GasEstimator: toml.KeySpecificGasEstimator{PriceMax: gasLanePriceWei},
 		})
 		sendEth(t, ownerKey, uni.backend, bhsKey.Address, 10)
 	}
 	keySpecificOverrides = append(keySpecificOverrides, toml.KeySpecific{
 		// Gas lane.
-		Key:          ptr[types.EIP55Address](vrfKey.EIP55Address),
+		Key:          new(vrfKey.EIP55Address),
 		GasEstimator: toml.KeySpecificGasEstimator{PriceMax: gasLanePriceWei},
 	})
 
 	keys = append(keys, ownerKey, vrfKey)
 
 	config, _ := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) {
 		simulatedOverrides(t, gasLanePriceWei, keySpecificOverrides...)(c, s)
-		c.EVM[0].MinIncomingConfirmations = ptr[uint32](2)
-		c.Feature.LogPoller = ptr(true)
-		c.EVM[0].FinalityDepth = ptr[uint32](2)
-		c.EVM[0].GasEstimator.LimitDefault = ptr(uint64(gasLimit))
+		c.EVM[0].MinIncomingConfirmations = new(uint32(2))
+		c.Feature.LogPoller = new(true)
+		c.EVM[0].FinalityDepth = new(uint32(2))
+		c.EVM[0].GasEstimator.LimitDefault = new(gasLimit)
 		c.EVM[0].LogPollInterval = commonconfig.MustNewDuration(time.Second)
 	})
 
@@ -86,11 +86,18 @@ func TestStartHeartbeats(t *testing.T) {
 		diff := heartbeatPeriod + 1*time.Second
 		t.Logf("Sleeping %.2f seconds before checking blockhash in BHS added by BHS_Heartbeats_Service\n", diff.Seconds())
 		time.Sleep(diff)
-		// storeEarliest in BHS contract stores blocktip - 256 in the Blockhash Store (BHS)
-		tipHeader, err := uni.backend.Client().HeaderByNumber(testutils.Context(t), nil)
-		require.NoError(t, err)
-		// the storeEarliest transaction will end up in a new block, hence the + 1 below.
-		blockNumberStored := tipHeader.Number.Uint64() - 256 + 1
-		verifyBlockhashStored(t, uni.coordinatorV2UniverseCommon, blockNumberStored)
+		// The heartbeat store tx may not reach the mempool before the first
+		// Commit under load, so we can't predict which block it mines in.
+		// Commit blocks and check current_tip-256 on each attempt until BHS
+		// has a blockhash stored at that offset.
+		require.Eventually(t, func() bool {
+			uni.backend.Commit()
+			tip, tipErr := uni.backend.Client().HeaderByNumber(testutils.Context(t), nil)
+			if tipErr != nil || tip == nil || tip.Number.Uint64() < 256 {
+				return false
+			}
+			_, err := uni.bhsContract.GetBlockhash(nil, new(big.Int).SetUint64(tip.Number.Uint64()-256))
+			return err == nil
+		}, testutils.WaitTimeoutCustom(t, 5*time.Minute), time.Second)
 	})
 }