Multiple RP with scheduler backporting from 1.18.5

api/api-rules/violation_exceptions.list

@@ -343,6 +343,9 @@ API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,EnvFrom
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,VolumeMounts
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,Volumes
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeDriver,TopologyKeys
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeList,Items
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeSpec,Drivers
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,AllowedTopologies
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,MountOptions
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClassList,Items
@@ -481,9 +484,17 @@ API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1al
 API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,TenantPartitionKubeConfigs
 API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyConfiguration,NodePortAddresses
 API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyIPVSConfiguration,ExcludeCIDRs
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,KubeSchedulerConfiguration,PluginConfig
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,PluginSet,Disabled
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,PluginSet,Enabled
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Extender,ManagedResources
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,CAData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,CertData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,KeyData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,LabelsPresence,Labels
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Extenders
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Predicates
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Priorities
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Resources
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Shape
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ServiceAffinity,Labels
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ClusterDNS
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,EnforceNodeAllocatable
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,TLSCipherSuites
@@ -681,6 +692,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,V
 API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,FlexVolumePluginDir
 API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,PersistentVolumeRecyclerConfiguration
 API rule violation: names_match,k8s.io/kube-proxy/config/v1alpha1,KubeProxyConfiguration,IPTables
+API rule violation: names_match,k8s.io/kube-scheduler/config/v1,Extender,EnableHTTPS
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,IPTablesDropBit
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,IPTablesMasqueradeBit
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ResolverConfig

build/kazel_generated.bzl

@@ -97,7 +97,9 @@ tags_values_pkgs = {"openapi-gen": {
         "staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1",
         "staging/src/k8s.io/kube-controller-manager/config/v1alpha1",
         "staging/src/k8s.io/kube-proxy/config/v1alpha1",
+        "staging/src/k8s.io/kube-scheduler/config/v1",
         "staging/src/k8s.io/kube-scheduler/config/v1alpha1",
+        "staging/src/k8s.io/kube-scheduler/config/v1alpha2",
         "staging/src/k8s.io/kubelet/config/v1beta1",
         "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1",
         "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2",
@@ -181,7 +183,9 @@ tags_pkgs_values = {"openapi-gen": {
     "staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1": ["true"],
     "staging/src/k8s.io/kube-controller-manager/config/v1alpha1": ["true"],
     "staging/src/k8s.io/kube-proxy/config/v1alpha1": ["true"],
+    "staging/src/k8s.io/kube-scheduler/config/v1": ["true"],
     "staging/src/k8s.io/kube-scheduler/config/v1alpha1": ["true"],
+    "staging/src/k8s.io/kube-scheduler/config/v1alpha2": ["true"],
     "staging/src/k8s.io/kubelet/config/v1beta1": ["true"],
     "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1": ["true"],
     "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2": ["true"],

cmd/cloud-controller-manager/app/core.go

@@ -36,12 +36,17 @@ import (
 
 func startCloudNodeController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
 	// Start the CloudNodeController
-	nodeController := cloudcontrollers.NewCloudNodeController(
+	nodeController, err := cloudcontrollers.NewCloudNodeController(
 		ctx.SharedInformers.Core().V1().Nodes(),
 		// cloud node controller uses existing cluster role from node-controller
 		ctx.ClientBuilder.ClientOrDie("node-controller"),
 		cloud,
-		ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration)
+		ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration,
+	)
+	if err != nil {
+		klog.Warningf("failed to start cloud node controller: %s", err)
+		return nil, false, nil
+	}
 
 	go nodeController.Run(stopCh)
 

cmd/controller-manager/app/serve.go

@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// File modified by cherrypick from kubernetes on 05/06/2021
 package app
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"net/http"
 	goruntime "runtime"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	apiserver "k8s.io/apiserver/pkg/server"
@@ -46,6 +48,7 @@ func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.Aut
 		handler = genericapifilters.WithAuthentication(handler, authenticationInfo.Authenticator, failedHandler, nil)
 	}
 	handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver)
+	handler = genericapifilters.WithCacheControl(handler)
 	handler = genericfilters.WithPanicRecovery(handler)
 
 	return handler

cmd/kube-controller-manager/app/BUILD

@@ -147,6 +147,7 @@ go_library(
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
         "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
+        "//staging/src/k8s.io/csi-translation-lib:go_default_library",
         "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library",
         "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
         "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library",

cmd/kube-controller-manager/app/core.go

@@ -23,6 +23,7 @@ package app
 
 import (
 	"fmt"
+	csitrans "k8s.io/csi-translation-lib"
 	"net"
 	"net/http"
 	"strings"
@@ -270,7 +271,8 @@ func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, err
 			ctx.InformerFactory.Core().V1().PersistentVolumes(),
 			ctx.InformerFactory.Storage().V1().StorageClasses(),
 			ctx.Cloud,
-			ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration))
+			ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
+			csitrans.New())
 
 		if expandControllerErr != nil {
 			return nil, true, fmt.Errorf("failed to start volume expand controller : %v", expandControllerErr)

cmd/kube-controller-manager/app/policy.go

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 The Kubernetes Authors.
+Copyright 2020 Authors of Arktos - file modified.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,14 +22,16 @@ limitations under the License.
 package app
 
 import (
+	"net/http"
+
+	"k8s.io/klog"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/scale"
 	"k8s.io/kubernetes/pkg/controller/disruption"
-
-	"net/http"
-
-	"k8s.io/klog"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 )
 
 func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error) {
@@ -42,6 +45,10 @@ func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error
 			resource, group+"/"+version)
 		return nil, false, nil
 	}
+	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodDisruptionBudget) {
+		klog.Infof("Refusing to start disruption because the PodDisruptionBudget feature is disabled")
+		return nil, false, nil
+	}
 
 	client := ctx.ClientBuilder.ClientOrDie("disruption-controller")
 	config := ctx.ClientBuilder.ConfigOrDie("disruption-controller")

cmd/kube-scheduler/BUILD

@@ -5,7 +5,7 @@ load(
     "go_binary",
     "go_library",
 )
-load("//pkg/version:def.bzl", "version_x_defs")
+load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "kube-scheduler",
@@ -20,10 +20,9 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
     deps = [
         "//cmd/kube-scheduler/app:go_default_library",
-        "//pkg/util/prometheusclientgo:go_default_library",
-        "//pkg/version/prometheus:go_default_library",
         "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
         "//staging/src/k8s.io/component-base/logs:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
     ],
 )

cmd/kube-scheduler/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- sig-scheduling-maintainers
+reviewers:
+- sig-scheduling
+labels:
+- sig/scheduling

cmd/kube-scheduler/app/BUILD

@@ -14,17 +14,16 @@ go_library(
         "//cmd/kube-scheduler/app/options:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/scheduler:go_default_library",
-        "//pkg/scheduler/algorithmprovider:go_default_library",
         "//pkg/scheduler/apis/config:go_default_library",
         "//pkg/scheduler/framework/v1alpha1:go_default_library",
         "//pkg/scheduler/metrics:go_default_library",
+        "//pkg/scheduler/profile:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/flag:go_default_library",
-        "//pkg/version:go_default_library",
         "//pkg/version/verflag:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/events/v1beta1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
@@ -34,12 +33,16 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/term:go_default_library",
-        "//staging/src/k8s.io/client-go/datapartition:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/events:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
         "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
-        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//staging/src/k8s.io/component-base/logs:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
+        "//staging/src/k8s.io/component-base/version:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

cmd/kube-scheduler/app/config/BUILD

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -12,7 +12,9 @@ go_library(
         "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/events:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
     ],
@@ -31,3 +33,13 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["config_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/github.com/google/go-cmp/cmp:go_default_library",
+    ],
+)

cmd/kube-scheduler/app/config/config.go

@@ -15,6 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// File modified by backporting scheduler 1.18.5 from kubernetes on 05/04/2021
 package config
 
 import (
@@ -23,15 +24,17 @@ import (
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/kubernetes/typed/events/v1beta1"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/events"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/record"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 )
 
 // Config has all the context to run a Scheduler
 type Config struct {
-	// config is the scheduler server's configuration object.
+	// ComponentConfig is the scheduler server's configuration object.
 	ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration
 
 	// LoopbackClientConfig is a config for a privileged loopback connection
@@ -43,16 +46,16 @@ type Config struct {
 	Authorization          apiserver.AuthorizationInfo
 	SecureServing          *apiserver.SecureServingInfo
 
-	// explictly define node informer from the resource provider client
-	ResourceProviderClient clientset.Interface
-	ResourceInformer       coreinformers.NodeInformer
-
 	Client          clientset.Interface
 	InformerFactory informers.SharedInformerFactory
 	PodInformer     coreinformers.PodInformer
-	EventClient     v1core.EventsGetter
-	Recorder        record.EventRecorder
-	Broadcaster     record.EventBroadcaster
+
+	// TODO: Remove the following after fully migrating to the new events api.
+	CoreEventClient v1core.EventsGetter
+	CoreBroadcaster record.EventBroadcaster
+
+	EventClient v1beta1.EventsGetter
+	Broadcaster events.EventBroadcaster
 
 	// LeaderElection is optional.
 	LeaderElection *leaderelection.LeaderElectionConfig

cmd/kube-scheduler/app/config/config_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Copyright 2020 Authors of Arktos - file modified.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// File modified by backporting scheduler 1.18.5 from kubernetes on 05/04/2021
+package config
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	apiserver "k8s.io/apiserver/pkg/server"
+)
+
+func TestConfigComplete(t *testing.T) {
+	scenarios := []struct {
+		name   string
+		want   *Config
+		config *Config
+	}{
+		{
+			name: "SetInsecureServingName",
+			want: &Config{
+				InsecureServing: &apiserver.DeprecatedInsecureServingInfo{
+					Name: "healthz",
+				},
+			},
+			config: &Config{
+				InsecureServing: &apiserver.DeprecatedInsecureServingInfo{},
+			},
+		},
+		{
+			name: "SetMetricsInsecureServingName",
+			want: &Config{
+				InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{
+					Name: "metrics",
+				},
+			},
+			config: &Config{
+				InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{},
+			},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			cc := scenario.config.Complete()
+
+			returnValue := cc.completedConfig.Config
+
+			if diff := cmp.Diff(scenario.want, returnValue); diff != "" {
+				t.Errorf("Complete(): Unexpected return value (-want, +got): %s", diff)
+			}
+		})
+	}
+}