Improve global expectation handling and synchronization

daemon/imon/converge_global_expect.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/opensvc/om3/v3/core/instance"
+	"github.com/opensvc/om3/v3/util/file"
 )
 
 // convergeGlobalExpectFromRemote set global expect from most recent global expect value
@@ -50,6 +51,20 @@ func (t *Manager) convergeGlobalExpectFromRemote() {
 
 func (t *Manager) isConvergedGlobalExpect() bool {
 	localUpdated := t.state.GlobalExpectUpdatedAt
+
+	// defines the expected instance monitors from scope nodes:
+	// If local instance has been created recently, it will take time to:
+	// 1- propagate the instance config file on peers
+	// 2- peer imon startup, push its state
+	// 3- peer imon state is propagated to local imon.
+	expectedInstanceMonitor := make(map[string]struct{})
+	for _, n := range t.scopeNodes {
+		if n == t.localhost {
+			continue
+		}
+		expectedInstanceMonitor[n] = struct{}{}
+	}
+
 	for s, v := range t.instMonitor {
 		if s == t.localhost {
 			err := fmt.Errorf("bug: isConvergedGlobalExpect detect unexpected localhost in internal instance monitor cache keys")
@@ -60,6 +75,38 @@ func (t *Manager) isConvergedGlobalExpect() bool {
 			t.log.Tracef("wait GlobalExpect propagation on %s", s)
 			return false
 		}
+		// we have a converged GlobalExpect (we don't expect any more propagation from this peer node)
+		delete(expectedInstanceMonitor, s)
+	}
+	if len(expectedInstanceMonitor) > 0 {
+		// expected peer instance monitors are still missing
+		switch t.state.GlobalExpect {
+		case instance.MonitorGlobalExpectPurged, instance.MonitorGlobalExpectDeleted:
+			// purge or deletion orchestration, we can ignore missing peer monitor states,
+			// peer instances may have been already deleted.
+		default:
+			for n := range expectedInstanceMonitor {
+				if _, ok := t.nodeMonitor[n]; ok {
+					t.log.Tracef("wait for initial GlobalExpect propagation on %s", n)
+					return false
+				}
+			}
+		}
+	}
+	if t.state.GlobalExpect != instance.MonitorGlobalExpectNone && t.state.GlobalExpect != instance.MonitorGlobalExpectInit {
+		if t.state.GlobalExpectUpdatedAt.Sub(t.instConfig.UpdatedAt) < time.Second {
+			// The instance config has been modified within the same second of the global expectation update.
+			// Take time to verify if we have the last instance config:
+			//     t.instConfig.UpdatedAt versus config file mod time
+			cfgFileModtime := file.ModTime(t.path.ConfigFile())
+			if cfgFileModtime.IsZero() {
+				return false
+			}
+			if cfgFileModtime.After(t.instConfig.UpdatedAt) {
+				t.log.Tracef("wait GlobalExpect propagation delayed: config file has been updated")
+				return false
+			}
+		}
 	}
 	return true
 }

daemon/imon/main.go

@@ -360,6 +360,12 @@ func (t *Manager) worker(initialNodes []string) {
 		t.instStatus[n] = *v
 	}
 
+	// force retrieval of possible lost global expectation from the peer
+	// instance monitor cache.
+	// This is usually checked on InstanceMonitorUpdated, but
+	// perhaps such events have been published before the subscription is running.
+	t.convergeGlobalExpectFromRemote()
+
 	t.delayTimer = time.NewTimer(time.Second)
 	if !t.delayTimer.Stop() {
 		<-t.delayTimer.C