Enable API server's watch termination grace period by default

By default, the API server doesn't terminate long-running watches
during shutdown: Active watch traffic is subject to regular HTTP server
shutdown behavior and will delay the shutdown until the overall HTTP
request timeout is reached. Previously, k0s relied on a fixed supervisor
stop timeout of five seconds, which did not account for this behavior
under realistic load.

Enable the API server's shutdown watch termination grace period by
default so active watch streams are drained during shutdown. Since watch
traffic accounts for most long-running API activity in normal clusters,
this allows for generally faster API server shutdowns, aligning with
k0s's requirement to promptly respond to shutdown requests from the
init system.

Derive the supervisor stop timeout from the API server flags, then
clamp it to the range of 5 to 20 seconds to stay within the typical init
system stop time budget. Set the watch termination grace period to the
stop timeout minus two seconds, if not explicitly specified by the user.
Finally, add a k0s controller command line flag to explicitly set the
API server's stop timeout and bypass its automatic calculation.

Signed-off-by: Tom Wieczorek <twieczorek@mirantis.com>

Author: twz123

cmd/controller/controller.go

@@ -302,6 +302,7 @@ func (c *command) start(ctx context.Context, flags *config.ControllerOptions, de
 		LogLevel:           c.LogLevels.KubeAPIServer,
 		Storage:            storageBackend,
 		EnableKonnectivity: enableKonnectivity,
+		StopTimeout:        flags.APIServerStopTimeout,
 
 		// If k0s reconciles the kubernetes endpoint, the API server shouldn't do it.
 		DisableEndpointReconciler: enableK0sEndpointReconciler,

cmd/controller/controller_test.go

@@ -50,6 +50,7 @@ Examples:
 	Note: Token can be passed either as a CLI argument, a flag, or an environment variable
 
 Flags:
+      --api-server-stop-timeout duration               time to wait for the API server to stop
   -c, --config string                                  config file, use '-' to read the config from stdin (default `+defaultConfigPath+`)
       --cri-socket string                              container runtime socket to use, default to internal containerd. Format: [remote|docker]:[path-to-socket]
       --data-dir string                                Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break! (default `+defaultDataDir+`)
@@ -82,3 +83,21 @@ Flags:
   -v, --verbose                                        Verbose logging (default true)
 `, out.String())
 }
+
+func TestControllerCmd_Flags(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("Running controllers is only supported on Linux")
+	}
+
+	t.Run("api-server-stop-timeout", func(t *testing.T) {
+		expected := `invalid argument "0s" for "--api-server-stop-timeout" flag: must be positive`
+		var stdout, stderr strings.Builder
+		underTest := cmd.NewRootCmd()
+		underTest.SetArgs([]string{"controller", "--api-server-stop-timeout", "0s"})
+		underTest.SetOut(&stdout)
+		underTest.SetErr(&stderr)
+		assert.ErrorContains(t, underTest.Execute(), expected)
+		assert.Empty(t, stdout.String())
+		assert.Equal(t, "Error: "+expected+"\n", stderr.String())
+	})
+}

cmd/install/controller_test.go

@@ -43,6 +43,7 @@ With the controller subcommand you can setup a single node cluster by running:
 	
 
 Flags:
+      --api-server-stop-timeout duration               time to wait for the API server to stop
   -c, --config string                                  config file, use '-' to read the config from stdin (default `+defaultConfigPath+`)
       --cri-socket string                              container runtime socket to use, default to internal containerd. Format: [remote|docker]:[path-to-socket]
       --data-dir string                                Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break! (default `+defaultDataDir+`)

pkg/component/controller/apiserver.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -38,6 +39,7 @@ type APIServer struct {
 	Storage                   manager.Component
 	EnableKonnectivity        bool
 	DisableEndpointReconciler bool
+	StopTimeout               time.Duration
 
 	supervisor     *supervisor.Supervisor
 	executablePath string
@@ -154,19 +156,59 @@ func (a *APIServer) Start(ctx context.Context) error {
 		args["endpoint-reconciler-type"] = "none"
 	}
 
+	stopTimeout := a.StopTimeout
+
+	// If the timeout hasn't been specified, do a
+	// best guess based on the API server flags.
+	if stopTimeout <= 0 {
+		requestTimeout := 1 * time.Minute
+		if value, ok := args["request-timeout"]; ok {
+			if parsed, err := time.ParseDuration(value); err == nil {
+				requestTimeout = parsed
+			}
+		}
+
+		watchTerminationGrace := 0 * time.Second
+		if value, ok := args["shutdown-watch-termination-grace-period"]; ok {
+			if parsed, err := time.ParseDuration(value); err == nil {
+				watchTerminationGrace = parsed
+			}
+		}
+
+		stopTimeout = max(requestTimeout, watchTerminationGrace) + (2 * time.Second)
+
+		// Clamp the timeout between 5 and 20 seconds. We can't wait for too long
+		// currently because the init system will likely kill the process otherwise.
+		stopTimeout = max(5*time.Second, min(stopTimeout, 20*time.Second))
+	}
+
+	// Enable the API server's watch-drain facility on shutdown, if that flag
+	// hasn't been specified by the user. Without this flag, the API server will
+	// almost always encounter the request timeout if anything is connected to
+	// it via client-go watches. These have a timeout of between five and ten
+	// minutes. Note that other types of long-running requests, such as log
+	// streams, can still prevent a timely shutdown. However, there's not much
+	// that can be done about them apart from setting a short request timeout.
+	if _, ok := args["shutdown-watch-termination-grace-period"]; !ok {
+		if gracePeriod := stopTimeout - 2*time.Second; gracePeriod > 0 {
+			args["shutdown-watch-termination-grace-period"] = gracePeriod.String()
+		}
+	}
+
 	var apiServerArgs []string
 	for name, value := range args {
 		apiServerArgs = append(apiServerArgs, fmt.Sprintf("--%s=%s", name, value))
 	}
 	apiServerArgs = append(apiServerArgs, a.ClusterConfig.Spec.API.RawArgs...)
 
 	a.supervisor = &supervisor.Supervisor{
-		Name:    kubeAPIComponentName,
-		BinPath: a.executablePath,
-		RunDir:  a.K0sVars.RunDir,
-		DataDir: a.K0sVars.DataDir,
-		Args:    apiServerArgs,
-		UID:     a.uid,
+		Name:        kubeAPIComponentName,
+		BinPath:     a.executablePath,
+		RunDir:      a.K0sVars.RunDir,
+		DataDir:     a.K0sVars.DataDir,
+		Args:        apiServerArgs,
+		UID:         a.uid,
+		TimeoutStop: stopTimeout,
 	}
 
 	etcdArgs, err := getEtcdArgs(a.ClusterConfig.Spec.Storage, a.K0sVars)

pkg/config/cli.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"runtime"
 	"slices"
@@ -57,6 +58,7 @@ type ControllerOptions struct {
 	EnableMetricsScraper            bool
 	KubeControllerManagerExtraArgs  string
 	FeatureGates                    featuregate.FeatureGates
+	APIServerStopTimeout            time.Duration
 
 	enableWorker, singleNode bool
 }
@@ -302,6 +304,7 @@ func GetControllerFlags(controllerOpts *ControllerOptions) *pflag.FlagSet {
 	flagset.StringVar(&controllerOpts.KubeControllerManagerExtraArgs, "kube-controller-manager-extra-args", "", "extra args for kube-controller-manager")
 	flagset.BoolVar(&controllerOpts.InitOnly, "init-only", false, "only initialize controller and exit")
 	flagset.Var(&controllerOpts.FeatureGates, "feature-gates", "feature gates to enable (comma separated list of key=value pairs)")
+	flagset.Var((*positiveDurationFlag)(&controllerOpts.APIServerStopTimeout), "api-server-stop-timeout", "time to wait for the API server to stop")
 	return flagset
 }
 
@@ -334,3 +337,39 @@ func GetCmdOpts(cobraCmd command) (*CLIOptions, error) {
 		K0sVars: k0sVars,
 	}, nil
 }
+
+type positiveDurationFlag time.Duration
+
+// Type implements [pflag.Value].
+func (f *positiveDurationFlag) Type() string {
+	return "duration"
+}
+
+// String implements [pflag.Value].
+func (f *positiveDurationFlag) String() string {
+	if *(*time.Duration)(f) <= 0 {
+		return ""
+	}
+
+	return (*time.Duration)(f).String()
+}
+
+// Set implements [pflag.Value].
+func (f *positiveDurationFlag) Set(value string) error {
+	if value == "" {
+		*(*time.Duration)(f) = 0
+		return nil
+	}
+
+	parsed, err := time.ParseDuration(value)
+	if err != nil {
+		return err
+	}
+
+	if parsed <= 0 {
+		return errors.New("must be positive")
+	}
+
+	*(*time.Duration)(f) = parsed
+	return nil
+}