Skip to content

Commit 8c2a56f

Browse files
askervinaskervin
committed
balloons: implement scheduling classes
Signed-off-by: Antti Kervinen <antti.kervinen@intel.com>
1 parent a301269 commit 8c2a56f

File tree

4 files changed

+137
-10
lines changed

4 files changed

+137
-10
lines changed

‎cmd/plugins/balloons/policy/balloons-policy.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ const (
4949
balloonKey = "balloon." + PolicyName + "." + kubernetes.ResmgrKeyNamespace
5050
// hideHyperthreadsKey is a pod annotation key for pod/container-specific hyperthread allowance.
5151
hideHyperthreadsKey = "hide-hyperthreads." + kubernetes.ResmgrKeyNamespace
52+
// schedulingClassKey is a pod annotation key for pod/container-specific scheduling class.
53+
schedulingClassKey = "scheduling-class." + kubernetes.ResmgrKeyNamespace
5254
// reservedBalloonDefName is the name in the reserved balloon definition.
5355
reservedBalloonDefName = "reserved"
5456
// defaultBalloonDefName is the name in the default balloon definition.
@@ -1509,6 +1511,7 @@ func (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, fre
15091511
func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
15101512
seenNames := map[string]struct{}{}
15111513
undefinedLoadClasses := map[string]struct{}{}
1514+
undefinedSchedulingClasses := map[string]struct{}{}
15121515
compositeBlnDefs := map[string]*BalloonDef{}
15131516
for _, blnDef := range bpoptions.BalloonDefs {
15141517
if blnDef.Name == "" {
@@ -1581,6 +1584,9 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
15811584
for _, load := range blnDef.Loads {
15821585
undefinedLoadClasses[load] = struct{}{}
15831586
}
1587+
if blnDef.SchedulingClass != "" {
1588+
undefinedSchedulingClasses[blnDef.SchedulingClass] = struct{}{}
1589+
}
15841590
}
15851591
for lcIndex, loadClass := range bpoptions.LoadClasses {
15861592
delete(undefinedLoadClasses, loadClass.Name)
@@ -1594,6 +1600,12 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {
15941600
if len(undefinedLoadClasses) > 0 {
15951601
return balloonsError("loads defined in balloonTypes but missing from loadClasses: %v", undefinedLoadClasses)
15961602
}
1603+
for _, schedClass := range bpoptions.SchedulingClasses {
1604+
delete(undefinedSchedulingClasses, schedClass.Name)
1605+
}
1606+
if len(undefinedSchedulingClasses) > 0 {
1607+
return balloonsError("schedulingClass(es) defined in balloonTypes but missing from schedulingClasses: %v", undefinedSchedulingClasses)
1608+
}
15971609
var circularCheck func(name string, seen map[string]int) error
15981610
circularCheck = func(name string, seen map[string]int) error {
15991611
if seen[name] > 0 {
@@ -2124,13 +2136,89 @@ func (bln *Balloon) updateGroups(c cache.Container, delta int) {
21242136
}
21252137
}
21262138

2139+
// applyProcessScheduling configures container's scheduling and IO priorities
2140+
func applyProcessScheduling(c cache.Container, sc *SchedulingClass) {
2141+
if sc == nil {
2142+
return
2143+
}
2144+
log.Debug(" - applying scheduling class %q to %s", sc.Name, c.PrettyName())
2145+
if sc.Policy != "" {
2146+
if pol, err := sc.Policy.ToNRI(); err == nil {
2147+
c.SetSchedulingPolicy(pol)
2148+
log.Debug(" - scheduling policy %q (%s)", sc.Policy, pol)
2149+
} else {
2150+
log.Debug(" - invalid scheduling policy %q in scheduling class %q: %v", sc.Policy, sc.Name, err)
2151+
}
2152+
}
2153+
if sc.Priority != nil {
2154+
c.SetSchedulingPriority(int32(*sc.Priority))
2155+
log.Debug(" - scheduling priority %d", *sc.Priority)
2156+
}
2157+
if len(sc.Flags) > 0 {
2158+
if flags, err := sc.Flags.ToNRI(); err == nil {
2159+
c.SetSchedulingFlags(flags)
2160+
log.Debug(" - scheduling flags %q", sc.Flags)
2161+
} else {
2162+
log.Debug(" - invalid scheduling flags %q in scheduling class %q: %v", sc.Flags, sc.Name, err)
2163+
}
2164+
}
2165+
if sc.Nice != nil {
2166+
c.SetSchedulingNice(int32(*sc.Nice))
2167+
log.Debug(" - nice value %d", *sc.Nice)
2168+
}
2169+
if sc.Runtime != nil {
2170+
c.SetSchedulingRuntime(*sc.Runtime)
2171+
log.Debug(" - scheduling runtime %d", *sc.Runtime)
2172+
}
2173+
if sc.Deadline != nil {
2174+
c.SetSchedulingDeadline(*sc.Deadline)
2175+
log.Debug(" - scheduling deadline %d", *sc.Deadline)
2176+
}
2177+
if sc.Period != nil {
2178+
c.SetSchedulingPeriod(*sc.Period)
2179+
log.Debug(" - scheduling period %d", *sc.Period)
2180+
}
2181+
if sc.IOClass != "" {
2182+
if ioClass, err := sc.IOClass.ToNRI(); err == nil {
2183+
c.SetSchedulingIOClass(ioClass)
2184+
log.Debug(" - IO class %q", sc.IOClass)
2185+
} else {
2186+
log.Debug(" - invalid IO class %q in scheduling class %q: %v", sc.IOClass, sc.Name, err)
2187+
}
2188+
}
2189+
if sc.IOPriority != nil {
2190+
c.SetSchedulingIOPriority(int32(*sc.IOPriority))
2191+
log.Debug(" - IO priority %d", *sc.IOPriority)
2192+
}
2193+
}
2194+
2195+
func (p *balloons) applyProcessProperties(c cache.Container, bln *Balloon) {
2196+
effSc := bln.Def.SchedulingClass
2197+
if annSc, annExists := c.GetEffectiveAnnotation(schedulingClassKey); annExists {
2198+
if annSc != effSc {
2199+
log.Debug(" - container %s overrides balloon scheduling class %q with annotation %q",
2200+
c.PrettyName(), effSc, annSc)
2201+
}
2202+
effSc = annSc
2203+
}
2204+
if effSc != "" {
2205+
for _, sc := range p.bpoptions.SchedulingClasses {
2206+
if sc.Name == effSc {
2207+
applyProcessScheduling(c, sc)
2208+
break
2209+
}
2210+
}
2211+
}
2212+
}
2213+
21272214
// assignContainer adds a container to a balloon
21282215
func (p *balloons) assignContainer(c cache.Container, bln *Balloon) {
21292216
log.Info("assigning container %s to balloon %s", c.PrettyName(), bln)
21302217
podID := c.GetPodID()
21312218
bln.PodIDs[podID] = append(bln.PodIDs[podID], c.GetID())
21322219
bln.updateGroups(c, 1)
21332220
p.updatePinning(bln)
2221+
p.applyProcessProperties(c, bln)
21342222
}
21352223

21362224
// dismissContainer removes a container from a balloon

‎cmd/plugins/balloons/policy/flags.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type (
2323
BalloonsOptions = cfgapi.Config
2424
BalloonDef = cfgapi.BalloonDef
2525
LoadClass = cfgapi.LoadClass
26+
SchedulingClass = cfgapi.SchedulingClass
2627
CPUTopologyLevel = cfgapi.CPUTopologyLevel
2728
)
2829

‎cmd/plugins/topology-aware/policy/mocks_test.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,33 @@ func (m *mockContainer) SetMemoryLimit(int64) {
479479
func (m *mockContainer) SetMemorySwap(int64) {
480480
panic("unimplemented")
481481
}
482+
func (m *mockContainer) SetSchedulingPolicy(nri.LinuxSchedulerPolicy) {
483+
panic("unimplemented")
484+
}
485+
func (m *mockContainer) SetSchedulingNice(int32) {
486+
panic("unimplemented")
487+
}
488+
func (m *mockContainer) SetSchedulingPriority(int32) {
489+
panic("unimplemented")
490+
}
491+
func (m *mockContainer) SetSchedulingFlags([]nri.LinuxSchedulerFlag) {
492+
panic("unimplemented")
493+
}
494+
func (m *mockContainer) SetSchedulingRuntime(uint64) {
495+
panic("unimplemented")
496+
}
497+
func (m *mockContainer) SetSchedulingDeadline(uint64) {
498+
panic("unimplemented")
499+
}
500+
func (m *mockContainer) SetSchedulingPeriod(uint64) {
501+
panic("unimplemented")
502+
}
503+
func (m *mockContainer) SetSchedulingIOClass(nri.IOPrioClass) {
504+
panic("unimplemented")
505+
}
506+
func (m *mockContainer) SetSchedulingIOPriority(int32) {
507+
panic("unimplemented")
508+
}
482509
func (m *mockContainer) GetPendingAdjustment() *nri.ContainerAdjustment {
483510
panic("unimplemented")
484511
}

‎pkg/apis/config/v1alpha1/resmgr/policy/schedulingclass.go‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,30 @@ type SchedulingClass struct {
2525
// SCHED_<NAME> translates to <name> etc.
2626
// +kubebuilder:validation:Enum=none;other;fifo;rr;batch;iso;idle;deadline
2727
// +kubebuilder:validation:Format:string
28-
Policy SchedulingPolicy `json:"policy,omitempty"`
29-
Priority *int `json:"priority,omitempty"`
28+
Policy SchedulingPolicy `json:"policy,omitempty"`
29+
// Priority is the Linux scheduling priority to use.
30+
// Valid range depends on the selected policy.
31+
// Refer to sched_setscheduler(2) for details.
32+
Priority *int `json:"priority,omitempty"`
3033
// Flags is a list of Linux scheduling flags to set.
3134
// SCHED_FLAG_<ORIG_NAME> translates to <orig-name> etc.
3235
// +kubebuilder:validation:Enum=reset-on-fork;reclaim;dl-overrun;keep-policy;keep-params;util-clamp-min;util-clamp-max
33-
Flags SchedulingFlags `json:"flags,omitempty"`
34-
Nice *int `json:"nice,omitempty"`
35-
Runtime *uint64 `json:"runtime,omitempty"`
36-
Deadline *uint64 `json:"deadline,omitempty"`
37-
Period *uint64 `json:"period,omitempty"`
38-
36+
Flags SchedulingFlags `json:"flags,omitempty"`
37+
// Nice is the Linux nice value to use.
38+
Nice *int `json:"nice,omitempty"`
39+
// Runtime is the Linux SCHED_DEADLINE runtime value to use (in microseconds).
40+
Runtime *uint64 `json:"runtime,omitempty"`
41+
// Deadline is the Linux SCHED_DEADLINE deadline value to use (in microseconds).
42+
Deadline *uint64 `json:"deadline,omitempty"`
43+
// Period is the Linux SCHED_DEADLINE period value to use (in microseconds).
44+
Period *uint64 `json:"period,omitempty"`
3945
// IOClass is the IO scheduling class to use.
46+
// IOPRIO_CLASS_<NAME> translates to <name>.
47+
// Refer to ioprio_set(2) and ionice(1) for details.
4048
// +kubebuilder:validation:Enum=none;rt;be;idle
41-
IOClass IOPriorityClass `json:"ioClass,omitempty"`
42-
IOPriority *int `json:"ioPriority,omitempty"`
49+
IOClass IOPriorityClass `json:"ioClass,omitempty"`
50+
// IOPriority is the IO priority within the selected IO class to use.
51+
// Valid range depends on the selected class.
52+
// Refer to ionice(1) for details.
53+
IOPriority *int `json:"ioPriority,omitempty"`
4354
}

0 commit comments

Comments
 (0)