topology-aware: initial scheduling class support.

cmd/plugins/balloons/policy/balloons-policy.go

@@ -49,6 +49,8 @@ const (
 	balloonKey = "balloon." + PolicyName + "." + kubernetes.ResmgrKeyNamespace
 	// hideHyperthreadsKey is a pod annotation key for pod/container-specific hyperthread allowance.
 	hideHyperthreadsKey = "hide-hyperthreads." + kubernetes.ResmgrKeyNamespace
+	// schedulingClassKey is a pod annotation key for pod/container-specific scheduling class.
+	schedulingClassKey = "scheduling-class." + kubernetes.ResmgrKeyNamespace
 	// reservedBalloonDefName is the name in the reserved balloon definition.
 	reservedBalloonDefName = "reserved"
 	// defaultBalloonDefName is the name in the default balloon definition.
@@ -1509,6 +1511,7 @@ func (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, fre
 func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
 	seenNames := map[string]struct{}{}
 	undefinedLoadClasses := map[string]struct{}{}
+	undefinedSchedulingClasses := map[string]struct{}{}
 	compositeBlnDefs := map[string]*BalloonDef{}
 	for _, blnDef := range bpoptions.BalloonDefs {
 		if blnDef.Name == "" {
@@ -1581,6 +1584,9 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
 		for _, load := range blnDef.Loads {
 			undefinedLoadClasses[load] = struct{}{}
 		}
+		if blnDef.SchedulingClass != "" {
+			undefinedSchedulingClasses[blnDef.SchedulingClass] = struct{}{}
+		}
 	}
 	for lcIndex, loadClass := range bpoptions.LoadClasses {
 		delete(undefinedLoadClasses, loadClass.Name)
@@ -1594,6 +1600,12 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
 	if len(undefinedLoadClasses) > 0 {
 		return balloonsError("loads defined in balloonTypes but missing from loadClasses: %v", undefinedLoadClasses)
 	}
+	for _, schedClass := range bpoptions.SchedulingClasses {
+		delete(undefinedSchedulingClasses, schedClass.Name)
+	}
+	if len(undefinedSchedulingClasses) > 0 {
+		return balloonsError("schedulingClass(es) defined in balloonTypes but missing from schedulingClasses: %v", undefinedSchedulingClasses)
+	}
 	var circularCheck func(name string, seen map[string]int) error
 	circularCheck = func(name string, seen map[string]int) error {
 		if seen[name] > 0 {
@@ -2124,13 +2136,89 @@ func (bln *Balloon) updateGroups(c cache.Container, delta int) {
 	}
 }
 
+// applyProcessScheduling configures container's scheduling and IO priorities
+func applyProcessScheduling(c cache.Container, sc *SchedulingClass) {
+	if sc == nil {
+		return
+	}
+	log.Debug("  - applying scheduling class %q to %s", sc.Name, c.PrettyName())
+	if sc.Policy != "" {
+		if pol, err := sc.Policy.ToNRI(); err == nil {
+			c.SetSchedulingPolicy(pol)
+			log.Debug("  - scheduling policy %q (%s)", sc.Policy, pol)
+		} else {
+			log.Debug("  - invalid scheduling policy %q in scheduling class %q: %v", sc.Policy, sc.Name, err)
+		}
+	}
+	if sc.Priority != nil {
+		c.SetSchedulingPriority(int32(*sc.Priority))
+		log.Debug("  - scheduling priority %d", *sc.Priority)
+	}
+	if len(sc.Flags) > 0 {
+		if flags, err := sc.Flags.ToNRI(); err == nil {
+			c.SetSchedulingFlags(flags)
+			log.Debug("  - scheduling flags %q", sc.Flags)
+		} else {
+			log.Debug("  - invalid scheduling flags %q in scheduling class %q: %v", sc.Flags, sc.Name, err)
+		}
+	}
+	if sc.Nice != nil {
+		c.SetSchedulingNice(int32(*sc.Nice))
+		log.Debug("  - nice value %d", *sc.Nice)
+	}
+	if sc.Runtime != nil {
+		c.SetSchedulingRuntime(*sc.Runtime)
+		log.Debug("  - scheduling runtime %d", *sc.Runtime)
+	}
+	if sc.Deadline != nil {
+		c.SetSchedulingDeadline(*sc.Deadline)
+		log.Debug("  - scheduling deadline %d", *sc.Deadline)
+	}
+	if sc.Period != nil {
+		c.SetSchedulingPeriod(*sc.Period)
+		log.Debug("  - scheduling period %d", *sc.Period)
+	}
+	if sc.IOClass != "" {
+		if ioClass, err := sc.IOClass.ToNRI(); err == nil {
+			c.SetSchedulingIOClass(ioClass)
+			log.Debug("  - IO class %q", sc.IOClass)
+		} else {
+			log.Debug("  - invalid IO class %q in scheduling class %q: %v", sc.IOClass, sc.Name, err)
+		}
+	}
+	if sc.IOPriority != nil {
+		c.SetSchedulingIOPriority(int32(*sc.IOPriority))
+		log.Debug("  - IO priority %d", *sc.IOPriority)
+	}
+}
+
+func (p *balloons) applyProcessProperties(c cache.Container, bln *Balloon) {
+	effSc := bln.Def.SchedulingClass
+	if annSc, annExists := c.GetEffectiveAnnotation(schedulingClassKey); annExists {
+		if annSc != effSc {
+			log.Debug("  - container %s overrides balloon scheduling class %q with annotation %q",
+				c.PrettyName(), effSc, annSc)
+		}
+		effSc = annSc
+	}
+	if effSc != "" {
+		for _, sc := range p.bpoptions.SchedulingClasses {
+			if sc.Name == effSc {
+				applyProcessScheduling(c, sc)
+				break
+			}
+		}
+	}
+}
+
 // assignContainer adds a container to a balloon
 func (p *balloons) assignContainer(c cache.Container, bln *Balloon) {
 	log.Info("assigning container %s to balloon %s", c.PrettyName(), bln)
 	podID := c.GetPodID()
 	bln.PodIDs[podID] = append(bln.PodIDs[podID], c.GetID())
 	bln.updateGroups(c, 1)
 	p.updatePinning(bln)
+	p.applyProcessProperties(c, bln)
 }
 
 // dismissContainer removes a container from a balloon

cmd/plugins/balloons/policy/flags.go

@@ -23,6 +23,7 @@ type (
 	BalloonsOptions  = cfgapi.Config
 	BalloonDef       = cfgapi.BalloonDef
 	LoadClass        = cfgapi.LoadClass
+	SchedulingClass  = cfgapi.SchedulingClass
 	CPUTopologyLevel = cfgapi.CPUTopologyLevel
 )
 

cmd/plugins/topology-aware/policy/mocks_test.go

@@ -479,6 +479,33 @@ func (m *mockContainer) SetMemoryLimit(int64) {
 func (m *mockContainer) SetMemorySwap(int64) {
 	panic("unimplemented")
 }
+func (m *mockContainer) SetSchedulingPolicy(nri.LinuxSchedulerPolicy) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingNice(int32) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingPriority(int32) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingFlags([]nri.LinuxSchedulerFlag) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingRuntime(uint64) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingDeadline(uint64) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingPeriod(uint64) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingIOClass(nri.IOPrioClass) {
+	panic("unimplemented")
+}
+func (m *mockContainer) SetSchedulingIOPriority(int32) {
+	panic("unimplemented")
+}
 func (m *mockContainer) GetPendingAdjustment() *nri.ContainerAdjustment {
 	panic("unimplemented")
 }

cmd/plugins/topology-aware/policy/pod-preferences.go

@@ -43,7 +43,8 @@ const (
 	keyHideHyperthreads = "hide-hyperthreads"
 	// annotation key for picking individual resources by topology hints
 	keyPickResourcesByHints = "pick-resources-by-hints"
-
+	// annotation key for scheduling class
+	keySchedulingClass = "scheduling-class." + kubernetes.ResmgrKeyNamespace
 	// effective annotation key for isolated CPU preference
 	preferIsolatedCPUsKey = "prefer-isolated-cpus" + "." + kubernetes.ResmgrKeyNamespace
 	// effective annotation key for shared CPU preference
@@ -236,6 +237,19 @@ func memoryTypePreference(pod cache.Pod, container cache.Container) memoryType {
 	return mtype
 }
 
+// schedulingClassPreference returns any annotated scheduling class preference
+// for the container.
+func schedulingClassPreference(ctr cache.Container) string {
+	value, ok := ctr.GetEffectiveAnnotation(keySchedulingClass)
+	if !ok {
+		return ""
+	}
+
+	log.Debug("%s: effective scheduling class preference %q", ctr.PrettyName(), value)
+
+	return value
+}
+
 // coldStartPreference figures out 'cold start' preferences for the container, IOW
 // if the container memory should be allocated for an initial 'cold start' period
 // from PMEM, and how long this initial period should be.

cmd/plugins/topology-aware/policy/pools.go

@@ -22,6 +22,7 @@ import (
 	"github.com/containers/nri-plugins/pkg/utils/cpuset"
 	corev1 "k8s.io/api/core/v1"
 
+	cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware"
 	"github.com/containers/nri-plugins/pkg/resmgr/cache"
 	libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
 	system "github.com/containers/nri-plugins/pkg/sysfs"
@@ -552,6 +553,19 @@ func (p *policy) applyGrant(grant Grant) {
 		}
 		container.SetCpusetMems(mems.MemsetString())
 	}
+
+	if name := schedulingClassPreference(container); name != "" {
+		if sc := p.getSchedulingClass(name); sc == nil {
+			log.Errorf("%s: unknown scheduling class %q", container.PrettyName(), name)
+		} else {
+			if grant.SharedPortion() > 0 {
+				log.Errorf("%s: ignoring scheduling class %q (container has shared CPUs)",
+					container.PrettyName(), name)
+			} else {
+				p.applySchedulingClass(container, sc)
+			}
+		}
+	}
 }
 
 // Release resources allocated by this grant.
@@ -625,6 +639,69 @@ func (p *policy) updateSharedAllocations(grant *Grant) {
 	}
 }
 
+// getSchedulingClass returns the named scheduling class.
+func (p *policy) getSchedulingClass(name string) *cfgapi.SchedulingClass {
+	for _, c := range opt.SchedulingClasses {
+		if c.Name == name {
+			return c
+		}
+	}
+	return nil
+}
+
+// applySchedulingClass applies a scheduling class to a container.
+func (p *policy) applySchedulingClass(c cache.Container, sc *cfgapi.SchedulingClass) {
+	log.Debug("  - applying scheduling class %q to %s", sc.Name, c.PrettyName())
+	if sc.Policy != "" {
+		if pol, err := sc.Policy.ToNRI(); err == nil {
+			c.SetSchedulingPolicy(pol)
+			log.Debug("  - scheduling policy %q (%s)", sc.Policy, pol)
+		} else {
+			log.Debug("  - invalid scheduling policy %q in scheduling class %q: %v", sc.Policy, sc.Name, err)
+		}
+	}
+	if sc.Priority != nil {
+		c.SetSchedulingPriority(int32(*sc.Priority))
+		log.Debug("  - scheduling priority %d", *sc.Priority)
+	}
+	if len(sc.Flags) > 0 {
+		if flags, err := sc.Flags.ToNRI(); err == nil {
+			c.SetSchedulingFlags(flags)
+			log.Debug("  - scheduling flags %q", sc.Flags)
+		} else {
+			log.Debug("  - invalid scheduling flags %q in scheduling class %q: %v", sc.Flags, sc.Name, err)
+		}
+	}
+	if sc.Nice != nil {
+		c.SetSchedulingNice(int32(*sc.Nice))
+		log.Debug("  - nice value %d", *sc.Nice)
+	}
+	if sc.Runtime != nil {
+		c.SetSchedulingRuntime(*sc.Runtime)
+		log.Debug("  - scheduling runtime %d", *sc.Runtime)
+	}
+	if sc.Deadline != nil {
+		c.SetSchedulingDeadline(*sc.Deadline)
+		log.Debug("  - scheduling deadline %d", *sc.Deadline)
+	}
+	if sc.Period != nil {
+		c.SetSchedulingPeriod(*sc.Period)
+		log.Debug("  - scheduling period %d", *sc.Period)
+	}
+	if sc.IOClass != "" {
+		if ioClass, err := sc.IOClass.ToNRI(); err == nil {
+			c.SetSchedulingIOClass(ioClass)
+			log.Debug("  - IO class %q", sc.IOClass)
+		} else {
+			log.Debug("  - invalid IO class %q in scheduling class %q: %v", sc.IOClass, sc.Name, err)
+		}
+	}
+	if sc.IOPriority != nil {
+		c.SetSchedulingIOPriority(int32(*sc.IOPriority))
+		log.Debug("  - IO priority %d", *sc.IOPriority)
+	}
+}
+
 func (p *policy) foreachGrant(pool Node, fn func(g Grant) bool) {
 	for _, g := range p.allocations.grants {
 		if !pool.IsSameNode(g.GetCPUNode()) {