Merge pull request #7188 from twz123/watch-termination-grace-period

Enable API server's watch termination grace period by default

Author: twz123

cmd/controller/controller.go

@@ -308,6 +308,7 @@ func (c *command) start(ctx context.Context, flags *config.ControllerOptions, de
 		LogLevel:           c.LogLevels.KubeAPIServer,
 		Storage:            storageBackend,
 		EnableKonnectivity: enableKonnectivity,
+		StopTimeout:        flags.APIServerStopTimeout,
 
 		// If k0s reconciles the kubernetes endpoint, the API server shouldn't do it.
 		DisableEndpointReconciler: enableK0sEndpointReconciler,

cmd/controller/controller_test.go

@@ -50,6 +50,7 @@ Examples:
 	Note: Token can be passed either as a CLI argument, a flag, or an environment variable
 
 Flags:
+      --api-server-stop-timeout duration               time to wait for the API server to stop
   -c, --config string                                  config file, use '-' to read the config from stdin (default `+defaultConfigPath+`)
       --cri-socket string                              container runtime socket to use, default to internal containerd. Format: [remote|docker]:[path-to-socket]
       --data-dir string                                Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break! (default `+defaultDataDir+`)
@@ -82,3 +83,21 @@ Flags:
   -v, --verbose                                        Verbose logging (default true)
 `, out.String())
 }
+
+func TestControllerCmd_Flags(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("Running controllers is only supported on Linux")
+	}
+
+	t.Run("api-server-stop-timeout", func(t *testing.T) {
+		expected := `invalid argument "0s" for "--api-server-stop-timeout" flag: must be positive`
+		var stdout, stderr strings.Builder
+		underTest := cmd.NewRootCmd()
+		underTest.SetArgs([]string{"controller", "--api-server-stop-timeout", "0s"})
+		underTest.SetOut(&stdout)
+		underTest.SetErr(&stderr)
+		assert.ErrorContains(t, underTest.Execute(), expected)
+		assert.Empty(t, stdout.String())
+		assert.Equal(t, "Error: "+expected+"\n", stderr.String())
+	})
+}

cmd/install/controller_test.go

@@ -43,6 +43,7 @@ With the controller subcommand you can setup a single node cluster by running:
 	
 
 Flags:
+      --api-server-stop-timeout duration               time to wait for the API server to stop
   -c, --config string                                  config file, use '-' to read the config from stdin (default `+defaultConfigPath+`)
       --cri-socket string                              container runtime socket to use, default to internal containerd. Format: [remote|docker]:[path-to-socket]
       --data-dir string                                Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break! (default `+defaultDataDir+`)

pkg/component/controller/apiserver.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -38,6 +39,7 @@ type APIServer struct {
 	Storage                   manager.Component
 	EnableKonnectivity        bool
 	DisableEndpointReconciler bool
+	StopTimeout               time.Duration
 
 	supervisor     *supervisor.Supervisor
 	executablePath string
@@ -154,19 +156,59 @@ func (a *APIServer) Start(ctx context.Context) error {
 		args["endpoint-reconciler-type"] = "none"
 	}
 
+	stopTimeout := a.StopTimeout
+
+	// If the timeout hasn't been specified, do a
+	// best guess based on the API server flags.
+	if stopTimeout <= 0 {
+		requestTimeout := 1 * time.Minute
+		if value, ok := args["request-timeout"]; ok {
+			if parsed, err := time.ParseDuration(value); err == nil {
+				requestTimeout = parsed
+			}
+		}
+
+		watchTerminationGrace := 0 * time.Second
+		if value, ok := args["shutdown-watch-termination-grace-period"]; ok {
+			if parsed, err := time.ParseDuration(value); err == nil {
+				watchTerminationGrace = parsed
+			}
+		}
+
+		stopTimeout = max(requestTimeout, watchTerminationGrace) + (2 * time.Second)
+
+		// Clamp the timeout between 5 and 20 seconds. We can't wait for too long
+		// currently because the init system will likely kill the process otherwise.
+		stopTimeout = max(5*time.Second, min(stopTimeout, 20*time.Second))
+	}
+
+	// Enable the API server's watch-drain facility on shutdown, if that flag
+	// hasn't been specified by the user. Without this flag, the API server will
+	// almost always encounter the request timeout if anything is connected to
+	// it via client-go watches. These have a timeout of between five and ten
+	// minutes. Note that other types of long-running requests, such as log
+	// streams, can still prevent a timely shutdown. However, there's not much
+	// that can be done about them apart from setting a short request timeout.
+	if _, ok := args["shutdown-watch-termination-grace-period"]; !ok {
+		if gracePeriod := stopTimeout - 2*time.Second; gracePeriod > 0 {
+			args["shutdown-watch-termination-grace-period"] = gracePeriod.String()
+		}
+	}
+
 	var apiServerArgs []string
 	for name, value := range args {
 		apiServerArgs = append(apiServerArgs, fmt.Sprintf("--%s=%s", name, value))
 	}
 	apiServerArgs = append(apiServerArgs, a.ClusterConfig.Spec.API.RawArgs...)
 
 	a.supervisor = &supervisor.Supervisor{
-		Name:    kubeAPIComponentName,
-		BinPath: a.executablePath,
-		RunDir:  a.K0sVars.RunDir,
-		DataDir: a.K0sVars.DataDir,
-		Args:    apiServerArgs,
-		UID:     a.uid,
+		Name:        kubeAPIComponentName,
+		BinPath:     a.executablePath,
+		RunDir:      a.K0sVars.RunDir,
+		DataDir:     a.K0sVars.DataDir,
+		Args:        apiServerArgs,
+		UID:         a.uid,
+		TimeoutStop: stopTimeout,
 	}
 
 	etcdArgs, err := getEtcdArgs(a.ClusterConfig.Spec.Storage, a.K0sVars)

pkg/config/cli.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"runtime"
 	"slices"
@@ -57,6 +58,7 @@ type ControllerOptions struct {
 	EnableMetricsScraper            bool
 	KubeControllerManagerExtraArgs  string
 	FeatureGates                    featuregate.FeatureGates
+	APIServerStopTimeout            time.Duration
 
 	enableWorker, singleNode bool
 }
@@ -302,6 +304,7 @@ func GetControllerFlags(controllerOpts *ControllerOptions) *pflag.FlagSet {
 	flagset.StringVar(&controllerOpts.KubeControllerManagerExtraArgs, "kube-controller-manager-extra-args", "", "extra args for kube-controller-manager")
 	flagset.BoolVar(&controllerOpts.InitOnly, "init-only", false, "only initialize controller and exit")
 	flagset.Var(&controllerOpts.FeatureGates, "feature-gates", "feature gates to enable (comma separated list of key=value pairs)")
+	flagset.Var((*positiveDurationFlag)(&controllerOpts.APIServerStopTimeout), "api-server-stop-timeout", "time to wait for the API server to stop")
 	return flagset
 }
 
@@ -334,3 +337,39 @@ func GetCmdOpts(cobraCmd command) (*CLIOptions, error) {
 		K0sVars: k0sVars,
 	}, nil
 }
+
+type positiveDurationFlag time.Duration
+
+// Type implements [pflag.Value].
+func (f *positiveDurationFlag) Type() string {
+	return "duration"
+}
+
+// String implements [pflag.Value].
+func (f *positiveDurationFlag) String() string {
+	if *(*time.Duration)(f) <= 0 {
+		return ""
+	}
+
+	return (*time.Duration)(f).String()
+}
+
+// Set implements [pflag.Value].
+func (f *positiveDurationFlag) Set(value string) error {
+	if value == "" {
+		*(*time.Duration)(f) = 0
+		return nil
+	}
+
+	parsed, err := time.ParseDuration(value)
+	if err != nil {
+		return err
+	}
+
+	if parsed <= 0 {
+		return errors.New("must be positive")
+	}
+
+	*(*time.Duration)(f) = parsed
+	return nil
+}