The website uses cookies. By using this site, you agree to our use of cookies as described in the Privacy Policy.
I Agree
blank_error__heading
blank_error__body
Text direction?

kubelet: add initial support for cgroupv2 #85218

Merged
Changes from all commits File filter... Jump to…
@@ -21,6 +21,7 @@ import (
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@@ -153,10 +154,11 @@ func (l *libcontainerAdapter) newManager(cgroups *libcontainerconfigs.Cgroup, pa
if !cgroupsystemd.UseSystemd() {
panic("systemd cgroup manager not available")
}
return &cgroupsystemd.LegacyManager{
Cgroups: cgroups,
Paths: paths,
}, nil
f, err := cgroupsystemd.NewSystemdCgroupsManager()

This comment has been minimized.

derekwaynecarr Mar 12, 2020
Member

does this manager work on v1 and v2 hosts?

This comment has been minimized.

derekwaynecarr Mar 12, 2020
Member

i need to check the runc code, but is this abstracing legacy from unified manager?

This comment has been minimized.

giuseppe Mar 12, 2020
Author Member

yes, NewSystemdCgroupsManager() checks for the cgroup version used:

https://github.com/opencontainers/runc/blob/master/libcontainer/cgroups/systemd/apply_systemd.go#L118-L136

If we are running on cgroup v1, it will use cgroupsystemd.LegacyManager

This comment has been minimized.

derekwaynecarr Mar 26, 2020
Member

thanks for pointer, and noting that i confirmed this is in the vendored version.

if err != nil {
return nil, err
}
return f(cgroups, paths), nil
}
return nil, fmt.Errorf("invalid cgroup manager configuration")
}
@@ -254,6 +256,7 @@ func (m *cgroupManagerImpl) Exists(name CgroupName) bool {
// in https://github.com/opencontainers/runc/issues/1440
// once resolved, we can remove this code.
whitelistControllers := sets.NewString("cpu", "cpuacct", "cpuset", "memory", "systemd")
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) || utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportNodePidsLimit) {
whitelistControllers.Insert("pids")
}
@@ -369,14 +372,31 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont
if resourceConfig.Memory != nil {
resources.Memory = *resourceConfig.Memory
}
if resourceConfig.CpuShares != nil {
resources.CpuShares = *resourceConfig.CpuShares
}
if resourceConfig.CpuQuota != nil {
resources.CpuQuota = *resourceConfig.CpuQuota
}
if resourceConfig.CpuPeriod != nil {
resources.CpuPeriod = *resourceConfig.CpuPeriod
if libcontainercgroups.IsCgroup2UnifiedMode() {
if resourceConfig.CpuShares != nil {
// Convert from the range [2-262144] to [1-10000]
resources.CpuWeight = (1 + ((*resourceConfig.CpuShares-2)*9999)/262142)

This comment has been minimized.

derekwaynecarr Mar 26, 2020
Member

minor nit: may be useful to move this conversion into a helper that we can then unit test in a follow-on.

This comment has been minimized.

}
quota := "max"
period := "100000"
if resourceConfig.CpuQuota != nil {
quota = strconv.FormatInt(*resourceConfig.CpuQuota, 10)
}
if resourceConfig.CpuPeriod != nil {
period = strconv.FormatUint(*resourceConfig.CpuPeriod, 10)
}
resources.CpuMax = fmt.Sprintf("%s %s", quota, period)
} else {
if resourceConfig.CpuShares != nil {
resources.CpuShares = *resourceConfig.CpuShares
}
if resourceConfig.CpuQuota != nil {
resources.CpuQuota = *resourceConfig.CpuQuota
}
if resourceConfig.CpuPeriod != nil {
resources.CpuPeriod = *resourceConfig.CpuPeriod
}
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) || utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportNodePidsLimit) {
if resourceConfig.PidsLimit != nil {
@@ -161,6 +161,10 @@ func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
return f, fmt.Errorf("%s - %v", localErr, err)
}
if cgroups.IsCgroup2UnifiedMode() {
return f, nil
}
expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
for _, mountPoint := range mountPoints {
if mountPoint.Type == cgroupMountType {
@@ -884,6 +888,11 @@ func getContainer(pid int) (string, error) {
return "", err
}
if cgroups.IsCgroup2UnifiedMode() {
c, _ := cgs[""]
return c, nil
}
cpu, found := cgs["cpu"]
if !found {
return "", cgroups.NewNotFoundError("cpu")
@@ -19,9 +19,11 @@ package cm
import (
"bufio"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
@@ -32,6 +34,7 @@ import (
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm/util"
)
const (
@@ -181,8 +184,8 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64)
return result
}
// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
// getCgroupSubsystemsV1 returns information about the mounted cgroup v1 subsystems
func getCgroupSubsystemsV1() (*CgroupSubsystems, error) {
// get all cgroup mounts.
allCgroups, err := libcontainercgroups.GetCgroupMounts(true)
if err != nil {
@@ -203,6 +206,41 @@ func GetCgroupSubsystems() (*CgroupSubsystems, error) {
}, nil
}
// getCgroupSubsystemsV2 returns information about the enabled cgroup v2 subsystems
func getCgroupSubsystemsV2() (*CgroupSubsystems, error) {
content, err := ioutil.ReadFile(filepath.Join(util.CgroupRoot, "cgroup.controllers"))
if err != nil {
return nil, err
}
mounts := []libcontainercgroups.Mount{}
controllers := strings.Fields(string(content))
mountPoints := make(map[string]string, len(controllers))
for _, controller := range controllers {
mountPoints[controller] = util.CgroupRoot
m := libcontainercgroups.Mount{
Mountpoint: util.CgroupRoot,
Root: util.CgroupRoot,
Subsystems: []string{controller},
}
mounts = append(mounts, m)
}
return &CgroupSubsystems{
This conversation was marked as resolved by giuseppe

This comment has been minimized.

AkihiroSuda Nov 13, 2019
Contributor

can we ditch "subsystem mountpoint" concept for v2 codes?

runc and containerd are trying to eliminate all "subsystem" concept: opencontainers/runc#2169 https://godoc.org/github.com/containerd/cgroups/v2

This comment has been minimized.

This comment has been minimized.

giuseppe Mar 9, 2020
Author Member

is this refactoring something that could be addressed later? I'd like to get the current version merged as soon as possible, so that we can already start using it

This comment has been minimized.

Mounts: mounts,
MountPoints: mountPoints,
}, nil
}
// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
if libcontainercgroups.IsCgroup2UnifiedMode() {
return getCgroupSubsystemsV2()
}
return getCgroupSubsystemsV1()
}
// getCgroupProcs takes a cgroup directory name as an argument
// reads through the cgroup's procs file and returns a list of tgid's.
// It returns an empty list if a procs file doesn't exists
@@ -23,19 +23,35 @@ import (
libcontainerutils "github.com/opencontainers/runc/libcontainer/utils"
)
const (
// CgroupRoot is the base path where cgroups are mounted
CgroupRoot = "/sys/fs/cgroup"
)
// GetPids gets pids of the desired cgroup
// Forked from opencontainers/runc/libcontainer/cgroup/fs.Manager.GetPids()
func GetPids(cgroupPath string) ([]int, error) {
dir, err := getCgroupPath(cgroupPath)
if err != nil {
return nil, err
dir := ""
if libcontainercgroups.IsCgroup2UnifiedMode() {
path, err := filepath.Rel("/", cgroupPath)
if err != nil {
return nil, err
}
dir = filepath.Join(CgroupRoot, path)
} else {
var err error
dir, err = getCgroupV1Path(cgroupPath)
if err != nil {
return nil, err
}
}
return libcontainercgroups.GetPids(dir)
}
// getCgroupPath gets the file path to the "devices" subsystem of the desired cgroup.
// getCgroupV1Path gets the file path to the "devices" subsystem of the desired cgroup.
// cgroupPath is the path in the cgroup hierarchy.
func getCgroupPath(cgroupPath string) (string, error) {
func getCgroupV1Path(cgroupPath string) (string, error) {
cgroupPath = libcontainerutils.CleanPath(cgroupPath)
mnt, root, err := libcontainercgroups.FindCgroupMountpointAndRoot(cgroupPath, "devices")
@@ -50,16 +66,16 @@ func getCgroupPath(cgroupPath string) (string, error) {
return filepath.Join(root, mnt, cgroupPath), nil
}
parentPath, err := getCgroupParentPath(mnt, root)
parentPath, err := getCgroupV1ParentPath(mnt, root)
if err != nil {
return "", err
}
return filepath.Join(parentPath, cgroupPath), nil
}
// getCgroupParentPath gets the parent filepath to this cgroup, for resolving relative cgroup paths.
func getCgroupParentPath(mountpoint, root string) (string, error) {
// getCgroupV1ParentPath gets the parent filepath to this cgroup, for resolving relative cgroup paths.
func getCgroupV1ParentPath(mountpoint, root string) (string, error) {
// Use GetThisCgroupDir instead of GetInitCgroupDir, because the creating
// process could in container and shared pid namespace with host, and
// /proc/1/cgroup could point to whole other world of cgroups.
ProTip! Use n and p to navigate between commits in a pull request.
Measure
Measure
Related Notes
Get a free MyMarkup account to save this article and view it later on any device.
Create account

End User License Agreement

Summary | 0 Annotation