Add monitoring for config digest missmatch

commit/metrics/prom.go

@@ -92,6 +92,10 @@ var (
 		Name: "ccip_commit_loopp_ccip_provider_supported",
 		Help: "Tracks whether LOOPP CCIP provider is supported for each chain family (1 = supported, 0 = not supported)",
 	}, []string{"chain_family"})
+	promCommitConfigDigestMismatch = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ccip_commit_config_digest_mismatch",
+		Help: "Reports whether the home chain config digest differs from the offramp config digest (1 = mismatch, 0 = match)",
+	}, []string{"chain_family", "chain_id"})
 )
 
 type PromReporter struct {
@@ -116,6 +120,9 @@ type PromReporter struct {
 	bhSequenceNumbers           metric.Int64Gauge
 	bhCommitLatestRound         metric.Int64Gauge
 	bhLooppProviderSupported    metric.Int64Gauge
+
+	configDigestMismatch   *prometheus.GaugeVec
+	bhConfigDigestMismatch metric.Int64Gauge
 }
 
 func NewPromReporter(
@@ -149,6 +156,10 @@ func NewPromReporter(
 	if err != nil {
 		return nil, fmt.Errorf("failed to register ccip_commit_loopp_ccip_provider_supported gauge: %w", err)
 	}
+	configDigestMismatch, err := bhClient.Meter.Int64Gauge("ccip_commit_config_digest_mismatch")
+	if err != nil {
+		return nil, fmt.Errorf("failed to register ccip_commit_config_digest_mismatch gauge: %w", err)
+	}
 
 	return &PromReporter{
 		lggr:        lggr,
@@ -162,6 +173,7 @@ func NewPromReporter(
 		sequenceNumbers:        promSequenceNumbers,
 		commitLatestRound:      promCommitLatestRoundID,
 		looppProviderSupported: promLooppCCIPProviderSupported,
+		configDigestMismatch:   promCommitConfigDigestMismatch,
 
 		processorLatencyHistogram: promProcessorLatencyHistogram,
 		processorOutputCounter:    promProcessorOutputCounter,
@@ -173,6 +185,7 @@ func NewPromReporter(
 		bhSequenceNumbers:           sequenceNumbers,
 		bhCommitLatestRound:         commitLatestRoundID,
 		bhLooppProviderSupported:    looppProviderSupported,
+		bhConfigDigestMismatch:      configDigestMismatch,
 	}, nil
 }
 
@@ -340,3 +353,15 @@ func (p *PromReporter) TrackLooppProviderSupported(looppCCIPProviderSupported ma
 		))
 	}
 }
+
+func (p *PromReporter) TrackConfigDigestMismatch(mismatch bool) {
+	var value float64
+	if mismatch {
+		value = 1
+	}
+	p.configDigestMismatch.WithLabelValues(p.chainFamily, p.chainID).Set(value)
+	p.bhConfigDigestMismatch.Record(context.Background(), int64(value), metric.WithAttributes(
+		attribute.String("chain_family", p.chainFamily),
+		attribute.String("chain_id", p.chainID),
+	))
+}

commit/metrics/reporter.go

@@ -27,11 +27,14 @@ type Reporter interface {
 
 	TrackProcessorLatency(processor string, method plugincommon.MethodType, latency time.Duration, err error)
 	TrackProcessorOutput(processor string, method plugincommon.MethodType, obs plugintypes.Trackable)
+
+	TrackConfigDigestMismatch(mismatch bool)
 }
 
 type CommitPluginReporter interface {
 	TrackObservation(obs committypes.Observation, round uint64)
 	TrackOutcome(outcome committypes.Outcome, round uint64)
+	TrackConfigDigestMismatch(mismatch bool)
 }
 
 type Noop struct{}
@@ -48,6 +51,8 @@ func (n *Noop) TrackProcessorLatency(string, plugincommon.MethodType, time.Durat
 
 func (n *Noop) TrackProcessorOutput(string, plugincommon.MethodType, plugintypes.Trackable) {}
 
+func (n *Noop) TrackConfigDigestMismatch(bool) {}
+
 var _ Reporter = &PromReporter{}
 var _ CommitPluginReporter = &PromReporter{}
 var _ merkleroot.MetricsReporter = &PromReporter{}

commit/plugin.go

@@ -291,6 +291,9 @@ func (p *Plugin) Observation(
 		return encoded, nil
 	}
 
+	// Check config digest every round and emit a mismatch metric.
+	p.trackConfigDigestMismatch(ctx, lggr)
+
 	prevOutcome, err := p.ocrTypeCodec.DecodeOutcome(outCtx.PreviousOutcome)
 	if err != nil {
 		return nil, fmt.Errorf("decode previous outcome: %w", err)
@@ -412,6 +415,21 @@ func (p *Plugin) ObserveFChain(lggr logger.Logger) map[cciptypes.ChainSelector]i
 	return fChain
 }
 
+func (p *Plugin) trackConfigDigestMismatch(ctx context.Context, lggr logger.Logger) {
+	configMatch, _, err := plugincommon.ConfigDigestsMatch(
+		ctx, p.ccipReader, consts.PluginTypeCommit, p.reportingCfg.ConfigDigest,
+	)
+	if err != nil {
+		lggr.Errorw("failed to check for config digest mismatch",
+			"err", err,
+			"homeChainConfigDigest", p.reportingCfg.ConfigDigest,
+			"pluginType", consts.PluginTypeCommit,
+		)
+		return
+	}
+	p.metricsReporter.TrackConfigDigestMismatch(!configMatch)
+}
+
 //nolint:gocyclo
 func (p *Plugin) Outcome(
 	ctx context.Context, outCtx ocr3types.OutcomeContext, q types.Query, aos []types.AttributedObservation,

commit/plugin_roledon_e2e_test.go

@@ -92,6 +92,8 @@ func TestPlugin_RoleDonE2E_NoPrevOutcome(t *testing.T) {
 		{
 			deps.ccipReader.EXPECT().DiscoverContracts(mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
 			deps.ccipReader.EXPECT().Sync(mock.Anything, mock.Anything).Return(nil)
+			deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")).
+				Return([32]byte{}, nil).Maybe()
 		}
 
 		// Source Chain Expectations - Makes sure only oracles that support specific source chains are reading them.
@@ -209,6 +211,8 @@ func TestPlugin_RoleDonE2E_RangesAndPricesSelectedPreviously(t *testing.T) {
 		{
 			deps.ccipReader.EXPECT().DiscoverContracts(mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
 			deps.ccipReader.EXPECT().Sync(mock.Anything, mock.Anything).Return(nil)
+			deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")).
+				Return([32]byte{}, nil).Maybe()
 		}
 
 		// Source Chain Expectations - Makes sure only oracles that support specific source chains are reading them.
@@ -372,6 +376,8 @@ func TestPlugin_RoleDonE2E_Discovery(t *testing.T) {
 					}, addresses)
 					return nil
 				})
+			deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")).
+				Return([32]byte{}, nil).Maybe()
 		}
 
 		p := s.newRoleDonTestPlugin(oracleID, true)

commit/plugin_test.go

@@ -205,6 +205,9 @@ func TestObservation_prices(t *testing.T) {
 
 			ccr.EXPECT().GetLatestPriceSeqNr(mock.Anything).Return(tc.onchainOcrSeqNum, tc.rpcErr).Maybe()
 
+			ccr.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")).
+				Return([32]byte{}, nil).Maybe()
+
 			tokenPriceObs := tokenprice.Observation{}
 			if tc.expObservedPrices {
 				tokenPriceObs.FeedTokenPrices = map[ccipocr3.UnknownEncodedAddress]ccipocr3.BigInt{

commit/report.go

@@ -1,9 +1,7 @@
 package commit
 
 import (
-	"bytes"
 	"context"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"maps"
@@ -212,16 +210,18 @@ func (p *Plugin) validateReport(
 		return cciptypes.CommitPluginReport{}, plugincommon.NewErrInvalidReport("dest chain not supported")
 	}
 
-	offRampConfigDigest, err := p.ccipReader.GetOffRampConfigDigest(ctx, consts.PluginTypeCommit)
+	match, offRampDigest, err := plugincommon.ConfigDigestsMatch(
+		ctx, p.ccipReader, consts.PluginTypeCommit, p.reportingCfg.ConfigDigest,
+	)
 	if err != nil {
-		err = plugincommon.NewErrValidatingReport(fmt.Errorf("get offramp config digest: %w", err))
-		return cciptypes.CommitPluginReport{}, plugincommon.NewErrValidatingReport(err)
+		return cciptypes.CommitPluginReport{},
+			plugincommon.NewErrValidatingReport(fmt.Errorf("check config digest: %w", err))
 	}
 
-	if !bytes.Equal(offRampConfigDigest[:], p.reportingCfg.ConfigDigest[:]) {
+	if !match {
 		lggr.Warnw("my config digest doesn't match offramp's config digest, not accepting report",
 			"myConfigDigest", p.reportingCfg.ConfigDigest,
-			"offRampConfigDigest", hex.EncodeToString(offRampConfigDigest[:]),
+			"offRampConfigDigest", plugincommon.FormatConfigDigest(offRampDigest),
 		)
 		return cciptypes.CommitPluginReport{}, plugincommon.NewErrInvalidReport("config digest mismatch")
 	}

execute/metrics/prom.go

@@ -101,6 +101,10 @@ var (
 		Name: "ccip_exec_loopp_ccip_provider_supported",
 		Help: "Tracks whether LOOPP CCIP provider is supported for each chain family (1 = supported, 0 = not supported)",
 	}, []string{"chain_family"})
+	promExecConfigDigestMismatch = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ccip_exec_config_digest_mismatch",
+		Help: "Reports whether the home chain config digest differs from the offramp config digest (1 = mismatch, 0 = match)",
+	}, []string{"chain_family", "chain_id"})
 )
 
 type PromReporter struct {
@@ -127,6 +131,9 @@ type PromReporter struct {
 	beholderProcessorErrors     metric.Int64Counter
 	bhExecLatestRound           metric.Int64Gauge
 	bhLooppProviderSupported    metric.Int64Gauge
+
+	configDigestMismatch   *prometheus.GaugeVec
+	bhConfigDigestMismatch metric.Int64Gauge
 }
 
 func NewPromReporter(
@@ -169,6 +176,10 @@ func NewPromReporter(
 	if err != nil {
 		return nil, fmt.Errorf("failed to register ccip_exec_loopp_ccip_provider_supported gauge: %w", err)
 	}
+	configDigestMismatch, err := bhClient.Meter.Int64Gauge("ccip_exec_config_digest_mismatch")
+	if err != nil {
+		return nil, fmt.Errorf("failed to register ccip_exec_config_digest_mismatch gauge: %w", err)
+	}
 
 	return &PromReporter{
 		lggr:        lggr,
@@ -184,6 +195,7 @@ func NewPromReporter(
 		processorErrors:           PromExecProcessorErrors,
 		latestRoundID:             PromExecLatestRoundID,
 		looppProviderSupported:    promLooppCCIPProviderSupported,
+		configDigestMismatch:      promExecConfigDigestMismatch,
 
 		bhLatencyHistogram:          latencyHistogram,
 		bhProcessorLatencyHistogram: processorLatencyHistogram,
@@ -193,6 +205,7 @@ func NewPromReporter(
 		beholderProcessorErrors:     processorErrors,
 		bhExecLatestRound:           execLatestRoundID,
 		bhLooppProviderSupported:    looppProviderSupported,
+		bhConfigDigestMismatch:      configDigestMismatch,
 	}, nil
 }
 
@@ -399,3 +412,15 @@ func (p *PromReporter) TrackLooppProviderSupported(looppCCIPProviderSupported ma
 		))
 	}
 }
+
+func (p *PromReporter) TrackConfigDigestMismatch(mismatch bool) {
+	var value float64
+	if mismatch {
+		value = 1
+	}
+	p.configDigestMismatch.WithLabelValues(p.chainFamily, p.chainID).Set(value)
+	p.bhConfigDigestMismatch.Record(context.Background(), int64(value), metric.WithAttributes(
+		attribute.String("chain_family", p.chainFamily),
+		attribute.String("chain_id", p.chainID),
+	))
+}

execute/metrics/reporter.go

@@ -19,6 +19,7 @@ type Reporter interface {
 	TrackLatency(state exectypes.PluginState, method plugincommon.MethodType, latency time.Duration, err error)
 	TrackProcessorOutput(string, plugincommon.MethodType, plugintypes.Trackable)
 	TrackProcessorLatency(processor string, method plugincommon.MethodType, latency time.Duration, err error)
+	TrackConfigDigestMismatch(mismatch bool)
 }
 
 type Noop struct{}
@@ -33,5 +34,7 @@ func (n *Noop) TrackProcessorOutput(string, plugincommon.MethodType, plugintypes
 
 func (n *Noop) TrackProcessorLatency(string, plugincommon.MethodType, time.Duration, error) {}
 
+func (n *Noop) TrackConfigDigestMismatch(bool) {}
+
 var _ Reporter = &Noop{}
 var _ Reporter = &PromReporter{}

execute/observation.go

@@ -1,9 +1,7 @@
 package execute
 
 import (
-	"bytes"
 	"context"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"slices"
@@ -19,6 +17,7 @@ import (
 	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"
 
 	"github.com/smartcontractkit/chainlink-ccip/execute/exectypes"
+	"github.com/smartcontractkit/chainlink-ccip/internal/plugincommon"
 	dt "github.com/smartcontractkit/chainlink-ccip/internal/plugincommon/discovery/discoverytypes"
 	"github.com/smartcontractkit/chainlink-ccip/pkg/logutil"
 )
@@ -56,6 +55,21 @@ func (p *Plugin) Observation(
 	}
 	lggr.Infow("decoded previous outcome", "previousOutcome", previousOutcome)
 
+	// Check config digest every round and emit a mismatch metric.
+	// Both home chain and offramp config digest reads are cached, safe to call every round.
+	configMatch, _, configDigestErr := plugincommon.ConfigDigestsMatch(
+		ctx, p.ccipReader, consts.PluginTypeExecute, p.reportingCfg.ConfigDigest,
+	)
+	if configDigestErr != nil {
+		lggr.Errorw("failed to check for config digest mismatch",
+			"err", configDigestErr,
+			"homeChainConfigDigest", p.reportingCfg.ConfigDigest,
+			"pluginType", consts.PluginTypeExecute,
+		)
+	} else {
+		p.observer.TrackConfigDigestMismatch(!configMatch)
+	}
+
 	// If the previous outcome was the filter state, and reports were built, mark the messages as inflight.
 	if previousOutcome.State == exectypes.Filter {
 		// the lane is invalid due to a config digest mismatch, skip updating
@@ -544,15 +558,17 @@ func (p *Plugin) getFilterObservation(
 }
 
 func (p *Plugin) checkConfigDigest(ctx context.Context) error {
-	offRampConfigDigest, err := p.ccipReader.GetOffRampConfigDigest(ctx, consts.PluginTypeExecute)
+	match, offRampDigest, err := plugincommon.ConfigDigestsMatch(
+		ctx, p.ccipReader, consts.PluginTypeExecute, p.reportingCfg.ConfigDigest,
+	)
 	if err != nil {
-		return fmt.Errorf("get offramp config digest: %w", err)
+		return err
 	}
 
-	if !bytes.Equal(offRampConfigDigest[:], p.reportingCfg.ConfigDigest[:]) {
+	if !match {
 		p.lggr.Warnw("home chain config digest doesn't match offramp's config digest, not starting",
 			"homeChainConfigDigest", p.reportingCfg.ConfigDigest,
-			"offRampConfigDigest", hex.EncodeToString(offRampConfigDigest[:]),
+			"offRampConfigDigest", plugincommon.FormatConfigDigest(offRampDigest),
 		)
 		return errOffRampConfigMismatch
 	}

execute/observation_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink-ccip/execute/exectypes"
 	"github.com/smartcontractkit/chainlink-ccip/execute/internal/cache"
+	execmetrics "github.com/smartcontractkit/chainlink-ccip/execute/metrics"
 	"github.com/smartcontractkit/chainlink-ccip/execute/tokendata/observer"
 	"github.com/smartcontractkit/chainlink-ccip/internal/mocks"
 	"github.com/smartcontractkit/chainlink-ccip/mocks/chainlink_common/ccipocr3"
@@ -58,6 +59,7 @@ func Test_Observation_CacheUpdate(t *testing.T) {
 		ocrTypeCodec:         ocrTypeCodec,
 		inflightMessageCache: cache.NewInflightMessageCache(10 * time.Minute),
 		ccipReader:           ccipReaderMock,
+		observer:             &execmetrics.Noop{},
 		reportingCfg: ocr3types.ReportingPluginConfig{
 			OracleID:     commontypes.OracleID(1),
 			ConfigDigest: configDigest,

execute/plugin_test.go

@@ -1215,6 +1215,10 @@ func TestPlugin_Observation_EligibilityCheckFailure(t *testing.T) {
 		GetRmnCurseInfo(mock.Anything).
 		Return(cciptypes.CurseInfo{}, nil).Maybe()
 
+	mockCCIPReader.EXPECT().
+		GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")).
+		Return([32]byte{}, nil).Maybe()
+
 	// Create a simplified plugin structure that will test the eligibility failure
 	// This removes the dependency on actual cache implementations
 	p := &Plugin{
@@ -1223,6 +1227,7 @@ func TestPlugin_Observation_EligibilityCheckFailure(t *testing.T) {
 		lggr:            lggr,
 		ocrTypeCodec:    ocrTypeCodec,
 		ccipReader:      mockCCIPReader,
+		observer:        &metrics.Noop{},
 		commitRootsCache: cache.NewCommitRootsCache(
 			lggr,
 			8*time.Hour,