背景
Today 3:44 PM 有同事 反馈 k8s 拉取镜像耗时很久，如下图所示：
从 log 可以看出，拉取镜像花费 2m8s，但是发起 Pulling 到 成功 pulled 镜像中间间隔 42min，原因何在？ 后面同事提供完整 log 截图后发现 waiting 时间就有 42m39s
代码分析
由于生产环境中的 k8s 版本为 v1.23.17, 因此我们基于此分支代码进行分析，进而寻求解决方案。
// EnsureImageExists pulls the image for the specified pod and container, and returns
// (imageRef, error message, error).
func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, string, error) {
..........
m.logIt(ref, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("Pulling image %q", container.Image), klog.Info)
startTime := time.Now()
pullChan := make(chan pullResult)
m.puller.pullImage(spec, pullSecrets, pullChan, podSandboxConfig)
imagePullResult := <-pullChan
if imagePullResult.err != nil {
m.logIt(ref, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, imagePullResult.err), klog.Warning)
m.backOff.Next(backOffKey, m.backOff.Clock.Now())
if imagePullResult.err == ErrRegistryUnavailable {
msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image)
return "", msg, imagePullResult.err
}
return "", imagePullResult.err.Error(), ErrImagePull
}
m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q in %v (%v including waiting)", container.Image, imagePullResult.pullDuration, time.Since(startTime)), klog.Info)
m.backOff.GC()
return imagePullResult.imageRef, "", nil
}
从代码（第22行）可以看出，从开启 pulling 到 pulled 结束一共花费 42m39s，实际镜像拉取时间为 2m8s，因此可以排除 Harbor 的原因。下面从 ImageManager 的初始化开始分析拉取镜像的流程。
// NewImageManager instantiates a new ImageManager object.
func NewImageManager(recorder record.EventRecorder, imageService kubecontainer.ImageService, imageBackOff *flowcontrol.Backoff, serialized bool, qps float32, burst int) ImageManager {
imageService = throttleImagePulling(imageService, qps, burst)
var puller imagePuller
if serialized {
puller = newSerialImagePuller(imageService)
} else {
puller = newParallelImagePuller(imageService) puller = newParallelImagePuller(imageService)
}
return &imageManager{
recorder: recorder,
imageService: imageService,
backOff: imageBackOff,
puller: puller,
}
}
从上面代码可以看出，初始化 ImageManager时通过指定 serialized 参数来决定是否是序列化拉取还是并发拉取（其实并发拉取并未正在实现，只是简单的起了一个 goroutine 来拉取镜像，并没有做并发限制，因此，如果同时拉取镜像太多会对节点造成很大压力），这个参数是由 kubelet 的 serializeImagePulls 来控制的，而/var/lib/kubelet/config.yaml 中 serializeImagePulls 默认值为 true。
serializeImagePulls: true
因此，我们只关心 newSerialImagePuller 的实现过程。
// Maximum number of image pull requests than can be queued.
const maxImagePullRequests = 10
type serialImagePuller struct {
imageService kubecontainer.ImageService
pullRequests chan *imagePullRequest
}
func newSerialImagePuller(imageService kubecontainer.ImageService) imagePuller {
imagePuller := &serialImagePuller{imageService, make(chan *imagePullRequest, maxImagePullRequests)}
go wait.Until(imagePuller.processImagePullRequests, time.Second, wait.NeverStop)
return imagePuller
}
type imagePullRequest struct {
spec kubecontainer.ImageSpec
pullSecrets []v1.Secret
pullChan chan<- pullResult
podSandboxConfig *runtimeapi.PodSandboxConfig
}
func (sip *serialImagePuller) pullImage(spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) {
sip.pullRequests <- &imagePullRequest{
spec: spec,
pullSecrets: pullSecrets,
pullChan: pullChan,
podSandboxConfig: podSandboxConfig,
}
}
func (sip *serialImagePuller) processImagePullRequests() {
for pullRequest := range sip.pullRequests {
startTime := time.Now()
imageRef, err := sip.imageService.PullImage(pullRequest.spec, pullRequest.pullSecrets, pullRequest.podSandboxConfig)
pullRequest.pullChan <- pullResult{
imageRef: imageRef,
err: err,
pullDuration: time.Since(startTime),
}
}
}
serialImagePuller 在初始化时会设置最大拉取镜像请求数的队列，puller 在收到拉取镜像的请求后会先将此请求放入此队列，后台依次从队列中取出拉取镜像请求并处理，这样如果请求数过多，或者拉取镜像比较耗时就会导致后面的拉取镜像请求一直阻塞。到这里，就已经清楚了为啥 waiting 时间会这么久。
如何解决
通过查看最新代码，发现已经实现了并发拉取，只需要设置以下参数即可，其最低支持版本为 v1.27
# Enable parallel image pulls
serializeImagePulls: false
# limit the number of parallel image pulls
maxParallelImagePulls: 10
func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) {
go func() {
if pip.tokens != nil {
pip.tokens <- struct{}{}
defer func() { <-pip.tokens }()
}
startTime := time.Now()
imageRef, err := pip.imageService.PullImage(ctx, spec, pullSecrets, podSandboxConfig)
var size uint64
if err == nil && imageRef != "" {
// Getting the image size with best effort, ignoring the error.
size, _ = pip.imageService.GetImageSize(ctx, spec)
}
pullChan <- pullResult{
imageRef: imageRef,
imageSize: size,
err: err,
pullDuration: time.Since(startTime),
}
}()
}
parallelImagePuller 在拉取镜像时会先获取 token，相当于控制同时拉取镜像的并发数，只有在获取到 token 之后才进行镜像的拉取，以上面设置的值为例，则支持同时并发拉取 10 个镜像，这样大大缓解 waiting 时间过长的问题。
此 feature 对应的 enhancement 链接为 https://github.com/kubernetes/enhancements/issues/3673
结论
- 解决方案
升级 k8s 版本
