// +kubebuilder:storageversion

// +kubebuilder:object:root=true  
// +kubebuilder:subresource:status  
// +kubebuilder:storageversion  
  
// DemoApplication is the Schema for the demoapplications API
type DemoApplication struct {  
    metav1.TypeMeta   `json:",inline"`  
    metav1.ObjectMeta `json:"metadata,omitempty"`  
  
    Spec   DemoApplicationSpec   `json:"spec,omitempty"`  
    Status DemoApplicationStatus `json:"status,omitempty"`  
}

func (r *DemoApplication) Hub() {}

func (a *DemoApplication) ConvertTo(dst conversion.Hub) error {
v1beta2App := dst.(*v1beta2.DemoApplication)
v1beta2App.ObjectMeta = a.ObjectMeta
v1beta2App.Spec.Image = a.Spec.Image
v1beta2App.Spec.Size = a.Spec.Replica
v1beta2App.Spec.Ports = a.Spec.Ports

return nil
}

func (a *DemoApplication) ConvertFrom(src conversion.Hub) error {
v1beta2App := src.(*v1beta2.DemoApplication)
a.ObjectMeta = v1beta2App.ObjectMeta
a.Spec.Image = v1beta2App.Spec.Image
a.Spec.Replica = v1beta2App.Spec.Size
a.Spec.Ports = v1beta2App.Spec.Ports

return nil
}

type DemoApplicationSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file

Image   string               `json:"image,omitempty"`
Replica *int32               `json:"replica,omitempty"`
Ports   []corev1.ServicePort `json:"ports,omitempty"`
}

func (r *DemoApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)

// TODO(user): your logic here

return ctrl.Result{}, nil
}

// 存在错误
return ctrl.Result{}, err

// 无错误，但重入队列
return ctrl.Result{Requeue: true}, nil

// 处理完成
return ctrl.Result{}, nil

// XX 时间后重新执行
return ctrl.Result{RequeueAfter: nextRun.Sub(r.Now())}, nil

application := &paascmftcomv1beta1.DemoApplication{}
if err := r.Get(ctx, req.NamespacedName, application); err != nil {
if apierrors.IsNotFound(err) {
log.Info("DemoApplication resource not found. Ignoring since it must be deleted")
return ctrl.Result{}, nil
}
log.Error(err, "Failed to get DemoApplication")
return ctrl.Result{}, err
}

found := &appsv1.Deployment{}
err := r.Get(ctx, req.NamespacedName, found)
if err != nil && apierrors.IsNotFound(err) {
// TODO 不存在，创建 Deployment 和 Service
} else if err != nil {
log.Error(err, "Failed to get Deployment")
return ctrl.Result{}, err
}

oldSpec := paascmftcomv1beta1.DemoApplicationSpec{}
if application.Annotations[AnnotationLastAppliedConfig] != "" {
if err = json.Unmarshal([]byte(application.Annotations[AnnotationLastAppliedConfig]), &oldSpec); err != nil {
return ctrl.Result{}, err
}
}

if !reflect.DeepEqual(application.Spec, oldSpec) {
// TODO 存在且有变化，更新 Deployment 和 Service
}

return ctrl.Result{}, nil

func (r *DemoApplicationReconciler) deploymentForDemoApplication(application *paascmftcomv1beta1.DemoApplication) *appsv1.Deployment {
labels := map[string]string{
"app": application.Name,
}

containerPorts := []corev1.ContainerPort
for _, port := range application.Spec.Ports {
containerPorts = append(containerPorts, corev1.ContainerPort{
ContainerPort: port.TargetPort.IntVal,
})
}

return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name:      application.Name,
Namespace: application.Namespace,
},
Spec: appsv1.DeploymentSpec{
Replicas: application.Spec.Replica,
Selector: &metav1.LabelSelector{
MatchLabels: labels,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name:  application.Name,
Image: application.Spec.Image,
Ports: containerPorts,
},
},
},
},
},
}
}

func (r *DemoApplicationReconciler) serviceForDemoApplication(application *paascmftcomv1beta1.DemoApplication) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name:      application.Name,
Namespace: application.Namespace,
},
Spec: corev1.ServiceSpec{
Ports: application.Spec.Ports,
Selector: map[string]string{
"app": application.Name,
},
},
}
}

dep := r.deploymentForDemoApplication(application)

log.Info("Creating a new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
if err := r.Create(ctx, dep); err != nil {
log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
return ctrl.Result{}, err
}

svc := r.serviceForDemoApplication(application)

log.Info("Creating a new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name)
if err := r.Create(ctx, svc); err != nil {
log.Error(err, "Failed to create new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name)
return ctrl.Result{}, err
}

cfg, _ := json.Marshal(application.Spec)
if application.Annotations != nil {
application.Annotations[AnnotationLastAppliedConfig] = string(cfg)
} else {
application.Annotations = map[string]string{
AnnotationLastAppliedConfig: string(cfg),
}
}

if err := r.Update(ctx, application); err != nil {
log.Error(err, "Failed to update DemoApplication", "DemoApplication.Namespace", application.Namespace, "DemoApplication.Name", application.Name)
return ctrl.Result{}, nil
}

// Requeue the request to ensure the Deployment is created
return ctrl.Result{RequeueAfter: time.Minute}, nil

newDep := r.deploymentForDemoApplication(application)
oldDep := &appsv1.Deployment{}
if err := r.Get(ctx, req.NamespacedName, oldDep); err != nil {
return ctrl.Result{}, err
}

oldDep.Spec = newDep.Spec
if err := r.Update(ctx, oldDep); err != nil {
return ctrl.Result{}, err
}

newSvc := r.serviceForDemoApplication(application)
oldSvc := &corev1.Service{}
if err := r.Get(ctx, req.NamespacedName, oldSvc); err != nil {
return ctrl.Result{}, err
}

oldSvc.Spec = newSvc.Spec
if err := r.Update(ctx, oldSvc); err != nil {
return ctrl.Result{}, err
}

cfg, _ := json.Marshal(application.Spec)
if application.Annotations != nil {
application.Annotations[AnnotationLastAppliedConfig] = string(cfg)
} else {
application.Annotations = map[string]string{
AnnotationLastAppliedConfig: string(cfg),
}
}

if err := r.Update(ctx, application); err != nil {
log.Error(err, "Failed to update DemoApplication", "DemoApplication.Namespace", application.Namespace, "DemoApplication.Name", application.Name)
return ctrl.Result{}, nil
}

ObjectMeta: metav1.ObjectMeta{
Name:      application.Name,
Namespace: application.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(application, schema.GroupVersionKind{
Group:   paascmftcomv1beta1.GroupVersion.Group,
Version: paascmftcomv1beta1.GroupVersion.Version,
Kind:    "DemoApplication",
}),
},
},

ctrl.SetControllerReference(application, dep, r.Scheme)

replicas := application.Spec.Replica
if *replicas != *found.Spec.Replicas {
found.Spec.Replicas = replicas
if err := r.Update(ctx, found); err != nil {
log.Error(err, "Failed to update Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
return ctrl.Result{}, err
}

return ctrl.Result{Requeue: true}, nil
}

func (r *DemoApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&paascmftcomv1beta1.DemoApplication{}).
Owns(&appsv1.Deployment{}).
Complete(r)
}

// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete

func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
// ...略
register("deployment", startDeploymentController)
// ...略
}

func startDeploymentController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
dc, err := deployment.NewDeploymentController(
controllerContext.InformerFactory.Apps().V1().Deployments(),
controllerContext.InformerFactory.Apps().V1().ReplicaSets(),
controllerContext.InformerFactory.Core().V1().Pods(),
controllerContext.ClientBuilder.ClientOrDie("deployment-controller"),
)
if err != nil {
return nil, true, fmt.Errorf("error creating Deployment controller: %v", err)
}
go dc.Run(ctx, int(controllerContext.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs))
return nil, true, nil
}

func (dc *DeploymentController) Run(ctx context.Context, workers int) {
defer utilruntime.HandleCrash()

// Start events processing pipeline.
dc.eventBroadcaster.StartStructuredLogging(0)
dc.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: dc.client.CoreV1().Events("")})
defer dc.eventBroadcaster.Shutdown()

defer dc.queue.ShutDown()

klog.InfoS("Starting controller", "controller", "deployment")
defer klog.InfoS("Shutting down controller", "controller", "deployment")

// 等待本地 deployment、rs 和 pod 缓存与服务器同步
if !cache.WaitForNamedCacheSync("deployment", ctx.Done(), dc.dListerSynced, dc.rsListerSynced, dc.podListerSynced) {
return
}

// 启动多个 worker 执行流程
for i := 0; i < workers; i++ {
go wait.UntilWithContext(ctx, dc.worker, time.Second)
}

<-ctx.Done()
}

func (dc *DeploymentController) worker(ctx context.Context) {
for dc.processNextWorkItem(ctx) {
}
}

func (dc *DeploymentController) processNextWorkItem(ctx context.Context) bool {
key, quit := dc.queue.Get()
if quit {
return false
}
defer dc.queue.Done(key)

err := dc.syncHandler(ctx, key.(string))
dc.handleErr(err, key)

return true
}

func (dc *XXController) processNextWorkItem(ctx context.Context) bool {

}

func NewDeploymentController(dInformer appsinformers.DeploymentInformer, rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, client clientset.Interface) (*DeploymentController, error) {
// ...略

dInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc:    dc.addDeployment,
UpdateFunc: dc.updateDeployment,
// This will enter the sync loop and no-op, because the deployment has been deleted from the store.
DeleteFunc: dc.deleteDeployment,
})
// ...略
}

func (dc *DeploymentController) addDeployment(obj interface{}) {
d := obj.(*apps.Deployment)
klog.V(4).InfoS("Adding deployment", "deployment", klog.KObj(d))
dc.enqueueDeployment(d)
}

dc.enqueueDeployment = dc.enqueue

func (dc *DeploymentController) enqueue(deployment *apps.Deployment) {
   key, err := controller.KeyFunc(deployment)
   if err != nil {
      utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", deployment, err))
      return
   }

   dc.queue.Add(key)
}

func (dc *DeploymentController) syncDeployment(ctx context.Context, key string) error {
// ...略

// 获取事件的 Deployment
deployment, err := dc.dLister.Deployments(namespace).Get(name)
if errors.IsNotFound(err) {
klog.V(2).InfoS("Deployment has been deleted", "deployment", klog.KRef(namespace, name))
return nil
}
if err != nil {
return err
}

// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
d := deployment.DeepCopy()

// 如果 deployment 的 selector 为空，则发布告警事件并返回
everything := metav1.LabelSelector{}
if reflect.DeepEqual(d.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
if d.Status.ObservedGeneration < d.Generation {
d.Status.ObservedGeneration = d.Generation
dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{})
}
return nil
}

// 获取 Deployment 对应的 ReplicaSet
rsList, err := dc.getReplicaSetsForDeployment(ctx, d)
if err != nil {
return err
}

// 获取 Deployment 的 Pod，使用 ReplicaSet 作为 Key 分组
// 返回值 podMap 的用途：
// 1. 检查 Pod 是否被 pod-template-hash 正确标识
// 2. 检查执行 Recreate Deployment 过程中，是否有旧 Pod 在运行
podMap, err := dc.getPodMapForDeployment(d, rsList)
if err != nil {
return err
}

if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(ctx, d, rsList)
}

// 在暂停或恢复 Deployment 时，使用 Unknown 状态更新 Deployment 状态。
// 这样可以保证有用户通过设置 progressDeadlineSeconds 恢复 Deployment 时不会超时
if err = dc.checkPausedConditions(ctx, d); err != nil {
return err
}

if d.Spec.Paused {
return dc.sync(ctx, d, rsList)
}

// rollback is not re-entrant in case the underlying replica sets are updated with a new
// revision so we should ensure that we won't proceed to update replica sets until we
// make sure that the deployment has cleaned up its rollback spec in subsequent enqueues.
if getRollbackTo(d) != nil {
return dc.rollback(ctx, d, rsList)
}

// 检查是否 scaling 事件
scalingEvent, err := dc.isScalingEvent(ctx, d, rsList)
if err != nil {
return err
}
if scalingEvent {
return dc.sync(ctx, d, rsList)
}

switch d.Spec.Strategy.Type {
case apps.RecreateDeploymentStrategyType:
return dc.rolloutRecreate(ctx, d, rsList, podMap)
case apps.RollingUpdateDeploymentStrategyType:
return dc.rolloutRolling(ctx, d, rsList)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}

func (dc *DeploymentController) sync(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
// 获取新 ReplicaSet 和历史所有的 ReplicaSet
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
if err != nil {
return err
}
// 尝试执行扩缩容
if err := dc.scale(ctx, d, newRS, oldRSs); err != nil {
// If we get an error while trying to scale, the deployment will be requeued
// so we can abort this resync
return err
}

// Clean up the deployment when it's paused and no rollback is in flight.
// 当前处于 Pause 状态但没在进行回滚时，进行清理 Deployment
if d.Spec.Paused && getRollbackTo(d) == nil {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}

allRSs := append(oldRSs, newRS)
// 同步 Deploymen 状态
return dc.syncDeploymentStatus(ctx, allRSs, newRS, d)
}

func (dc *DeploymentController) isScalingEvent(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) (bool, error) {
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
if err != nil {
return false, err
}
allRSs := append(oldRSs, newRS)
logger := klog.FromContext(ctx)
for _, rs := range controller.FilterActiveReplicaSets(allRSs) {
desired, ok := deploymentutil.GetDesiredReplicasAnnotation(logger, rs)
if !ok {
continue
}
// 注解的值和 Deployment 的期望值不同
if desired != *(d.Spec.Replicas) {
return true, nil
}
}
return false, nil
}

func (dc *DeploymentController) getAllReplicaSetsAndSyncRevision(
ctx context.Context, 
d *apps.Deployment, 
rsList []*apps.ReplicaSet, 
createIfNotExisted bool) (*apps.ReplicaSet, []*apps.ReplicaSet, error) {
//...
}

func (dc *DeploymentController) scale(ctx context.Context, deployment *apps.Deployment, newRS *apps.ReplicaSet, oldRSs []*apps.ReplicaSet) error {

// 获取活跃或最新的 ReplicaSet，有多个活跃 RS 时返回 nil；
// 如果只有一个活跃的 ReplicaSet 就把这个 RS 副本数扩容到 Deployment 配置的；
// 如果无活跃的 RS，则扩容最新的 RS
if activeOrLatest := deploymentutil.FindActiveOrLatest(newRS, oldRSs); activeOrLatest != nil {
// 如果 RS 副本数已经和 Deployment 配置的相同就退出
if *(activeOrLatest.Spec.Replicas) == *(deployment.Spec.Replicas) {
return nil
}
// 执行 scale 操作
_, _, err := dc.scaleReplicaSetAndRecordEvent(ctx, activeOrLatest, *(deployment.Spec.Replicas), deployment)
return err
}

// 如果新的 RS 已经饱和则旧 RS 应该全部缩容到 0
if deploymentutil.IsSaturated(deployment, newRS) {
for _, old := range controller.FilterActiveReplicaSets(oldRSs) {
if _, _, err := dc.scaleReplicaSetAndRecordEvent(ctx, old, 0, deployment); err != nil {
return err
}
}
return nil
}

// 在滚动更新时，同时存在老的 RS 和新的 RS，需要适当控制 RS 的扩缩容以保证不超过 MaxSurge
if deploymentutil.IsRollingUpdate(deployment) {
allRSs := controller.FilterActiveReplicaSets(append(oldRSs, newRS))
allRSsReplicas := deploymentutil.GetReplicaCountForReplicaSets(allRSs)

allowedSize := int32(0)
if *(deployment.Spec.Replicas) > 0 {
allowedSize = *(deployment.Spec.Replicas) + deploymentutil.MaxSurge(*deployment)
}

// 需要增加的 Pod 数量，负数则为减少。调整的数量需要合适的分布到活跃的 RS 中
deploymentReplicasToAdd := allowedSize - allRSsReplicas

// 缩放方向决定了在我们试图缩放相同大小 RS 的情况下会发生什么。
// 当进行扩容时，应操作更新的 RS；缩容时先操作更老的 RS
var scalingOperation string
switch {
case deploymentReplicasToAdd > 0:
sort.Sort(controller.ReplicaSetsBySizeNewer(allRSs))
scalingOperation = "up"

case deploymentReplicasToAdd < 0:
sort.Sort(controller.ReplicaSetsBySizeOlder(allRSs))
scalingOperation = "down"
}

// 遍历所有活跃 RS，计算每个 RS 的副本数量
deploymentReplicasAdded := int32(0)
nameToSize := make(map[string]int32)
for i := range allRSs {
rs := allRSs[i]

// 如果有需要调整的 Pod 就进行调整，否则保持
if deploymentReplicasToAdd != 0 {
proportion := deploymentutil.GetProportion(rs, *deployment, deploymentReplicasToAdd, deploymentReplicasAdded)

nameToSize[rs.Name] = *(rs.Spec.Replicas) + proportion
deploymentReplicasAdded += proportion
} else {
nameToSize[rs.Name] = *(rs.Spec.Replicas)
}
}

// 更新 RS
for i := range allRSs {
rs := allRSs[i]

// Add/remove any leftovers to the largest replica set.
// 第一个 RS
if i == 0 && deploymentReplicasToAdd != 0 {
// 把多余的变化应用到第一个 RS 上
leftover := deploymentReplicasToAdd - deploymentReplicasAdded
nameToSize[rs.Name] = nameToSize[rs.Name] + leftover
if nameToSize[rs.Name] < 0 {
nameToSize[rs.Name] = 0
}
}

// 更新 RS
if _, _, err := dc.scaleReplicaSet(ctx, rs, nameToSize[rs.Name], deployment, scalingOperation); err != nil {
// Return as soon as we fail, the deployment is requeued
return err
}
}
}
return nil
}

func (dc *DeploymentController) rolloutRolling(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
// 获取 RS，如果不存在新 RS 就创建
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, true)
if err != nil {
return err
}
allRSs := append(oldRSs, newRS)

// 尝试进行扩容操作
scaledUp, err := dc.reconcileNewReplicaSet(ctx, allRSs, newRS, d)
if err != nil {
return err
}
if scaledUp {
// Update DeploymentStatus
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 尝试进行缩容操作
scaledDown, err := dc.reconcileOldReplicaSets(ctx, allRSs, controller.FilterActiveReplicaSets(oldRSs), newRS, d)
if err != nil {
return err
}
if scaledDown {
// Update DeploymentStatus
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

if deploymentutil.DeploymentComplete(d, &d.Status) {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}

// Sync deployment status
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

func (dc *DeploymentController) rolloutRecreate(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet, podMap map[types.UID][]*v1.Pod) error {
// 如果不存在新 RS，在缩容前不创建
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
if err != nil {
return err
}
allRSs := append(oldRSs, newRS)
activeOldRSs := controller.FilterActiveReplicaSets(oldRSs)

// 对 RS 进行缩容
scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(ctx, activeOldRSs, d)
if err != nil {
return err
}
if scaledDown {
// Update DeploymentStatus.
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 缩容没完成，还有 Pod 在运行，先等着（先跳过）
if oldPodsRunning(newRS, oldRSs, podMap) {
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 如果不存在新 RS，就创建
if newRS == nil {
newRS, oldRSs, err = dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, true)
if err != nil {
return err
}
allRSs = append(oldRSs, newRS)
}

// 扩容新 RS
if _, err := dc.scaleUpNewReplicaSetForRecreate(ctx, newRS, d); err != nil {
return err
}

if util.DeploymentComplete(d, &d.Status) {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}

// Sync deployment status.
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

func (dc *DeploymentController) rolloutRolling(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
// 获取 RS，当不更新时，newRS 取原先的 RS
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, true)
if err != nil {
return err
}
allRSs := append(oldRSs, newRS)

// 如果容量不变，不扩容
scaledUp, err := dc.reconcileNewReplicaSet(ctx, allRSs, newRS, d)
if err != nil {
return err
}
if scaledUp {
// Update DeploymentStatus
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 缩容操作。如果活跃的旧 RS 为空，不缩容
scaledDown, err := dc.reconcileOldReplicaSets(ctx, allRSs, controller.FilterActiveReplicaSets(oldRSs), newRS, d)
if err != nil {
return err
}
if scaledDown {
// Update DeploymentStatus
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 更新状态
if deploymentutil.DeploymentComplete(d, &d.Status) {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}

// Sync deployment status
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

func (dc *DeploymentController) rolloutRecreate(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet, podMap map[types.UID][]*v1.Pod) error {
// 当不更新时，newRS 取原先的 RS
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
if err != nil {
return err
}
allRSs := append(oldRSs, newRS)
// oldRSs 中没有活跃 RS，这里为空
activeOldRSs := controller.FilterActiveReplicaSets(oldRSs)

// 对 RS 进行缩容，activeOldRSs 为空，实际不缩容
scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(ctx, activeOldRSs, d)
if err != nil {
return err
}
if scaledDown {
// Update DeploymentStatus.
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// 在进行的 Pod 属于当前 newRS 的，跳过
if oldPodsRunning(newRS, oldRSs, podMap) {
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

// newRS 不会为 nil
if newRS == nil {
newRS, oldRSs, err = dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, true)
if err != nil {
return err
}
allRSs = append(oldRSs, newRS)
}

// 副本数没变化，实际没扩容
if _, err := dc.scaleUpNewReplicaSetForRecreate(ctx, newRS, d); err != nil {
return err
}

// 更新状态
if util.DeploymentComplete(d, &d.Status) {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}

// Sync deployment status.
return dc.syncRolloutStatus(ctx, allRSs, newRS, d)
}

func defaultObjectRestarter(obj runtime.Object) ([]byte, error) {  
    switch obj := obj.(type) {  
    case *extensionsv1beta1.Deployment:  
       if obj.Spec.Paused {  
          return nil, errors.New("can't restart paused deployment (run rollout resume first)")  
       }  
       if obj.Spec.Template.ObjectMeta.Annotations == nil {  
          obj.Spec.Template.ObjectMeta.Annotations = make(map[string]string)  
       }  
       obj.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339)  
       return runtime.Encode(scheme.Codecs.LegacyCodec(extensionsv1beta1.SchemeGroupVersion), obj)
// ...略
}

func newThrottledUpgradeAwareProxyHandler(location *url.URL, transport http.RoundTripper, wrapTransport, upgradeRequired bool, responder rest.Responder) *proxy.UpgradeAwareHandler {
handler := proxy.NewUpgradeAwareHandler(location, transport, wrapTransport, upgradeRequired, proxy.NewErrorResponder(responder))
handler.MaxBytesPerSec = capabilities.Get().PerConnectionBandwidthLimitBytesPerSec
return handler
}

func (h *UpgradeAwareHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if h.tryUpgrade(w, req) {
return
}
// 强制升级但升级失败返回错误响应
if h.UpgradeRequired {
h.Responder.Error(w, req, errors.NewBadRequest("Upgrade request required"))
return
}

loc := *h.Location
loc.RawQuery = req.URL.RawQuery

// If original request URL ended in '/', append a '/' at the end of the
// of the proxy URL
if !strings.HasSuffix(loc.Path, "/") && strings.HasSuffix(req.URL.Path, "/") {
loc.Path += "/"
}

// 处理重定向
proxyRedirect := proxyRedirectsforRootPath(loc.Path, w, req)
if proxyRedirect {
return
}

// 如果没配置自定义 Transport 或开启了 WrapTransport 就使用默认 Transport
if h.Transport == nil || h.WrapTransport {
h.Transport = h.defaultProxyTransport(req.URL, h.Transport)
}

// WithContext creates a shallow clone of the request with the same context.
newReq := req.WithContext(req.Context())
newReq.Header = utilnet.CloneHeader(req.Header)
if !h.UseRequestLocation {
newReq.URL = &loc
}
if h.UseLocationHost {
// exchanging req.Host with the backend location is necessary for backends that act on the HTTP host header (e.g. API gateways),
// because req.Host has preference over req.URL.Host in filling this header field
newReq.Host = h.Location.Host
}

// create the target location to use for the reverse proxy
reverseProxyLocation := &url.URL{Scheme: h.Location.Scheme, Host: h.Location.Host}
if h.AppendLocationPath {
reverseProxyLocation.Path = h.Location.Path
}

// 构造代理
proxy := httputil.NewSingleHostReverseProxy(reverseProxyLocation)
proxy.Transport = h.Transport
proxy.FlushInterval = h.FlushInterval
proxy.ErrorLog = log.New(noSuppressPanicError{}, "", log.LstdFlags)
if h.RejectForwardingRedirects {
oldModifyResponse := proxy.ModifyResponse
proxy.ModifyResponse = func(response *http.Response) error {
code := response.StatusCode
if code >= 300 && code <= 399 && len(response.Header.Get("Location")) > 0 {
// close the original response
response.Body.Close()
msg := "the backend attempted to redirect this request, which is not permitted"
// replace the response
*response = http.Response{
StatusCode:    http.StatusBadGateway,
Status:        fmt.Sprintf("%d %s", response.StatusCode, http.StatusText(response.StatusCode)),
Body:          io.NopCloser(strings.NewReader(msg)),
ContentLength: int64(len(msg)),
}
} else {
if oldModifyResponse != nil {
if err := oldModifyResponse(response); err != nil {
return err
}
}
}
return nil
}
}
if h.Responder != nil {
// if an optional error interceptor/responder was provided wire it
// the custom responder might be used for providing a unified error reporting
// or supporting retry mechanisms by not sending non-fatal errors to the clients
proxy.ErrorHandler = h.Responder.Error
}

// 转发代理请求
proxy.ServeHTTP(w, newReq)
}

func IsUpgradeRequest(req *http.Request) bool {
for _, h := range req.Header[http.CanonicalHeaderKey(HeaderConnection)] {
if strings.Contains(strings.ToLower(h), strings.ToLower(HeaderUpgrade)) {
return true
}
}
return false
}

func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Request) bool {
if !httpstream.IsUpgradeRequest(req) {
klog.V(6).Infof("Request was not an upgrade")
return false
}
// ...略

clone := utilnet.CloneRequest(req)
// ...略
backendConn, err = h.DialForUpgrade(clone)
if err != nil {
klog.V(6).Infof("Proxy connection error: %v", err)
h.Responder.Error(w, req, err)
return true
}
defer backendConn.Close()
// ...略
}

func (h *UpgradeAwareHandler) DialForUpgrade(req *http.Request) (net.Conn, error) {
if h.UpgradeTransport == nil {
return dial(req, h.Transport)
}
updatedReq, err := h.UpgradeTransport.WrapRequest(req)
if err != nil {
return nil, err
}
return dial(updatedReq, h.UpgradeTransport)
}

func dial(req *http.Request, transport http.RoundTripper) (net.Conn, error) {
conn, err := dialURL(req.Context(), req.URL, transport)
if err != nil {
return nil, fmt.Errorf("error dialing backend: %v", err)
}

if err = req.Write(conn); err != nil {
conn.Close()
return nil, fmt.Errorf("error sending request: %v", err)
}

return conn, err
}

func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Request) bool {
// ...略

backendConn, err = h.DialForUpgrade(clone)
if err != nil {
klog.V(6).Infof("Proxy connection error: %v", err)
h.Responder.Error(w, req, err)
return true
}
defer backendConn.Close()

// determine the http response code from the backend by reading from rawResponse+backendConn
backendHTTPResponse, headerBytes, err := getResponse(io.MultiReader(bytes.NewReader(rawResponse), backendConn))
if err != nil {
klog.V(6).Infof("Proxy connection error: %v", err)
h.Responder.Error(w, req, err)
return true
}

if len(headerBytes) > len(rawResponse) {
// we read beyond the bytes stored in rawResponse, update rawResponse to the full set of bytes read from the backend
rawResponse = headerBytes
}
// ...略
}

if backendHTTPResponse.StatusCode != http.StatusSwitchingProtocols && backendHTTPResponse.StatusCode < 400 {
err := fmt.Errorf("invalid upgrade response: status code %d", backendHTTPResponse.StatusCode)
klog.Errorf("Proxy upgrade error: %v", err)
h.Responder.Error(w, req, err)
return true
}

if backendHTTPResponse.StatusCode != http.StatusSwitchingProtocols {
// If the backend did not upgrade the request, echo the response from the backend to the client and return, closing the connection.
klog.V(6).Infof("Proxy upgrade error, status code %d", backendHTTPResponse.StatusCode)
// set read/write deadlines
deadline := time.Now().Add(10 * time.Second)
backendConn.SetReadDeadline(deadline)
requestHijackedConn.SetWriteDeadline(deadline)
// write the response to the client
err := backendHTTPResponse.Write(requestHijackedConn)
if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
klog.Errorf("Error proxying data from backend to client: %v", err)
}
// Indicate we handled the request
return true
}

requestHijacker, ok := w.(http.Hijacker)
if !ok {
klog.V(6).Infof("Unable to hijack response writer: %T", w)
h.Responder.Error(w, req, fmt.Errorf("request connection cannot be hijacked: %T", w))
return true
}
requestHijackedConn, _, err := requestHijacker.Hijack()
if err != nil {
klog.V(6).Infof("Unable to hijack response: %v", err)
h.Responder.Error(w, req, fmt.Errorf("error hijacking connection: %v", err))
return true
}
defer requestHijackedConn.Close()

if len(rawResponse) > 0 {
klog.V(6).Infof("Writing %d bytes to hijacked connection", len(rawResponse))
if _, err = requestHijackedConn.Write(rawResponse); err != nil {
utilruntime.HandleError(fmt.Errorf("Error proxying response from backend to client: %v", err))
}
}

writerComplete := make(chan struct{})
readerComplete := make(chan struct{})

go func() {
var writer io.WriteCloser
if h.MaxBytesPerSec > 0 {
writer = flowrate.NewWriter(backendConn, h.MaxBytesPerSec)
} else {
writer = backendConn
}
_, err := io.Copy(writer, requestHijackedConn)
if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
klog.Errorf("Error proxying data from client to backend: %v", err)
}
close(writerComplete)
}()

go func() {
var reader io.ReadCloser
if h.MaxBytesPerSec > 0 {
reader = flowrate.NewReader(backendConn, h.MaxBytesPerSec)
} else {
reader = backendConn
}
_, err := io.Copy(requestHijackedConn, reader)
if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
klog.Errorf("Error proxying data from backend to client: %v", err)
}
close(readerComplete)
}()

// Wait for one half the connection to exit. Once it does the defer will
// clean up the other half of the connection.
select {
case <-writerComplete:
case <-readerComplete:
}
klog.V(6).Infof("Disconnecting from backend proxy %s\n  Headers: %v", &location, clone.Header)

func NewControllerManagerCommand() *cobra.Command {
s, err := options.NewKubeControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}

cmd := &cobra.Command{
Use: "kube-controller-manager",
// ...略过
RunE: func(cmd *cobra.Command, args []string) error {
// 略过
return Run(c.Complete(), wait.NeverStop)
},
// ...略过
}

fs := cmd.Flags()
// 命令行参数注册，略过
return cmd
}

run := func(ctx context.Context, startSATokenController InitFunc, initializersFunc ControllerInitializersFunc) {
// ...
}

// No leader election, run directly
if !c.ComponentConfig.Generic.LeaderElection.LeaderElect {
ctx, _ := wait.ContextForChannel(stopCh)
run(ctx, saTokenControllerInitFunc, NewControllerInitializers)
return nil
}

// Start the main lock
go leaderElectAndRun(c, id, electionChecker,
c.ComponentConfig.Generic.LeaderElection.ResourceLock,
c.ComponentConfig.Generic.LeaderElection.ResourceName,
leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
initializersFunc := NewControllerInitializers
if leaderMigrator != nil {
// If leader migration is enabled, we should start only non-migrated controllers
//  for the main lock.
initializersFunc = createInitializersFunc(leaderMigrator.FilterFunc, leadermigration.ControllerNonMigrated)
klog.Info("leader migration: starting main controllers.")
}
run(ctx, startSATokenController, initializersFunc)
},
OnStoppedLeading: func() {
klog.ErrorS(nil, "leaderelection lost")
klog.FlushAndExit(klog.ExitFlushTimeout, 1)
},
})

//...省略
<-stopCh

run := func(ctx context.Context, startSATokenController InitFunc, initializersFunc ControllerInitializersFunc) {
controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done())
if err != nil {
klog.Fatalf("error building controller context: %v", err)
}
// 初始化要运行的 Controller
controllerInitializers := initializersFunc(controllerContext.LoopMode)
// 启动所有 Controller
if err := StartControllers(ctx, controllerContext, startSATokenController, controllerInitializers, unsecuredMux, healthzHandler); err != nil {
klog.Fatalf("error starting controllers: %v", err)
}

// 启动 informer
controllerContext.InformerFactory.Start(stopCh)
controllerContext.ObjectOrMetadataInformerFactory.Start(stopCh)
close(controllerContext.InformersStarted)

<-ctx.Done()
}

func StartControllers(ctx context.Context, controllerCtx ControllerContext, startSATokenController InitFunc, controllers map[string]InitFunc,
unsecuredMux *mux.PathRecorderMux, healthzHandler *controllerhealthz.MutableHealthzHandler) error {
// 先启动 SATokenController，因为这个 Controller 会为其它 Controller 构造 Token
// 如果启动失败则中止启动流程
if startSATokenController != nil {
if _, _, err := startSATokenController(ctx, controllerCtx); err != nil {
return err
}
}

// Initialize the cloud provider with a reference to the clientBuilder only after token controller
// has started in case the cloud provider uses the client builder.
if controllerCtx.Cloud != nil {
controllerCtx.Cloud.Initialize(controllerCtx.ClientBuilder, ctx.Done())
}

var controllerChecks []healthz.HealthChecker

// 遍历 Controller 列表
for controllerName, initFn := range controllers {
if !controllerCtx.IsControllerEnabled(controllerName) {
klog.Warningf("%q is disabled", controllerName)
continue
}
// 加点时间间隔
time.Sleep(wait.Jitter(controllerCtx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))

// 启动 Controller 
klog.V(1).Infof("Starting %q", controllerName)
ctrl, started, err := initFn(ctx, controllerCtx)
if err != nil {
klog.Errorf("Error starting %q", controllerName)
return err
}
if !started {
klog.Warningf("Skipping %q", controllerName)
continue
}
// ...省略配置 Check 代码
controllerChecks = append(controllerChecks, check)

klog.Infof("Started %q", controllerName)
}

healthzHandler.AddHealthChecker(controllerChecks...)

return nil
}