Add apiserver-monitor problem daemon

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+
+- Add `apiserver-monitor` problem daemon to check connectivity to the Kubernetes API server.
+
 ## [0.8.7] - 2020-02-18
 
 ### Added

README.md

@@ -63,6 +63,7 @@ List of supported problem daemons types:
 | [SystemLogMonitor](https://github.com/kubernetes/node-problem-detector/tree/master/pkg/systemlogmonitor) | KernelDeadlock ReadonlyFilesystem FrequentKubeletRestart FrequentDockerRestart FrequentContainerdRestart | A system log monitor monitors system log and reports problems and metrics according to predefined rules. | [filelog](https://github.com/kubernetes/node-problem-detector/blob/master/config/kernel-monitor-filelog.json), [kmsg](https://github.com/kubernetes/node-problem-detector/blob/master/config/kernel-monitor.json), [kernel](https://github.com/kubernetes/node-problem-detector/blob/master/config/kernel-monitor-counter.json) [abrt](https://github.com/kubernetes/node-problem-detector/blob/master/config/abrt-adaptor.json) [systemd](https://github.com/kubernetes/node-problem-detector/blob/master/config/systemd-monitor-counter.json) | disable_system_log_monitor
 | [SystemStatsMonitor](https://github.com/kubernetes/node-problem-detector/tree/master/pkg/systemstatsmonitor) | None(Could be added in the future) | A system stats monitor for node-problem-detector to collect various health-related system stats as metrics. See the proposal [here](https://docs.google.com/document/d/1SeaUz6kBavI283Dq8GBpoEUDrHA2a795xtw0OvjM568/edit). | [system-stats-monitor](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json) | disable_system_stats_monitor
 | [CustomPluginMonitor](https://github.com/kubernetes/node-problem-detector/tree/master/pkg/custompluginmonitor) | On-demand(According to users configuration), existing example: NTPProblem | A custom plugin monitor for node-problem-detector to invoke and check various node problems with user-defined check scripts. See the proposal [here](https://docs.google.com/document/d/1jK_5YloSYtboj-DtfjmYKxfNnUxCAvohLnsH5aGCAYQ/edit#). | [example](https://github.com/kubernetes/node-problem-detector/blob/4ad49bbd84b8ced45ac825eac01ec93d9235935e/config/custom-plugin-monitor.json) | disable_custom_plugin_monitor
+| [APIServerMonitor](https://github.com/kubernetes/node-problem-detector/tree/master/pkg/apiservermonitor) | APIServerUnreachable | An apiserver monitor for node-problem-detector to check connectivity to the Kubernetes API server. | [apiserver-monitor](https://github.com/kubernetes/node-problem-detector/blob/master/config/apiserver-monitor.json) | disable_apiserver_monitor
 | [HealthChecker](https://github.com/kubernetes/node-problem-detector/tree/master/pkg/healthchecker) | KubeletUnhealthy ContainerRuntimeUnhealthy| A health checker for node-problem-detector to check kubelet and container runtime health. | [kubelet](https://github.com/kubernetes/node-problem-detector/blob/master/config/health-checker-kubelet.json) [docker](https://github.com/kubernetes/node-problem-detector/blob/master/config/health-checker-docker.json) [containerd](https://github.com/kubernetes/node-problem-detector/blob/master/config/health-checker-containerd.json) |
 
 # Exporter
@@ -104,6 +105,12 @@ certain backends. Some of them can be disabled at compile-time using a build tag
   Node problem detector will start a separate custom plugin monitor for each configuration.  You can
   use different custom plugin monitors to monitor different node problems.
 
+#### For APIServer Monitor
+
+* `--config.apiserver-monitor`: List of paths to apiserver monitor config files, comma-separated, e.g.
+  [config/apiserver-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/apiserver-monitor.json).
+  Node problem detector will start a separate apiserver monitor for each configuration.
+
 
 #### For Health Checkers
 

cmd/nodeproblemdetector/node_problem_detector.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/node-problem-detector/pkg/problemdetector"
 	"k8s.io/node-problem-detector/pkg/problemmetrics"
 	"k8s.io/node-problem-detector/pkg/types"
+	"k8s.io/node-problem-detector/pkg/util"
 	otelutil "k8s.io/node-problem-detector/pkg/util/otel"
 	"k8s.io/node-problem-detector/pkg/version"
 )
@@ -46,6 +47,12 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro
 	npdo.SetConfigFromDeprecatedOptionsOrDie()
 	npdo.ValidOrDie()
 
+	var err error
+	util.GlobalKubeClientConfig, err = util.NewKubeClientConfig(npdo.ApiServerOverride, npdo.QPS, npdo.Burst)
+	if err != nil {
+		klog.Warningf("Failed to create kube client config: %v", err)
+	}
+
 	// Initialize exporters first to set up the OpenTelemetry readers.
 	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(ctx, npdo); ke != nil {

cmd/nodeproblemdetector/problemdaemonplugins/apiserver_monitor_plugin.go

@@ -0,0 +1,23 @@
+//go:build !disable_apiserver_monitor
+
+/*
+Copyright 2026 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package problemdaemonplugins
+
+import (
+	_ "k8s.io/node-problem-detector/pkg/apiservermonitor"
+)

config/apiserver-monitor.json

@@ -0,0 +1,13 @@
+{
+  "invokeInterval": "30m",
+  "timeout": "30s",
+  "source": "apiserver-monitor",
+  "metricsReporting": true,
+  "conditions": [
+    {
+      "type": "APIServerUnreachable",
+      "reason": "APIServerIsReachable",
+      "message": "apiserver is reachable"
+    }
+  ]
+}

pkg/apiservermonitor/apiserver_monitor.go

@@ -0,0 +1,287 @@
+/*
+Copyright 2026 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiservermonitor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"slices"
+	"time"
+
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	kube_rest "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+
+	"k8s.io/node-problem-detector/pkg/problemdaemon"
+	"k8s.io/node-problem-detector/pkg/problemmetrics"
+	"k8s.io/node-problem-detector/pkg/types"
+	"k8s.io/node-problem-detector/pkg/util"
+	"k8s.io/node-problem-detector/pkg/util/tomb"
+)
+
+// APIServerMonitorName is the name of the apiserver problem daemon.
+const APIServerMonitorName = "apiserver-monitor"
+
+type apiServerMonitor struct {
+	configPath string
+	config     MonitorConfig
+	kubeConfig *kube_rest.Config
+	conditions []types.Condition
+	statusChan chan *types.Status
+	tomb       *tomb.Tomb
+}
+
+func init() {
+	problemdaemon.Register(
+		APIServerMonitorName,
+		types.ProblemDaemonHandler{
+			CreateProblemDaemonOrDie: NewAPIServerMonitorOrDie,
+			CmdOptionDescription:     "Set to config file paths.",
+		})
+}
+
+// NewAPIServerMonitorOrDie creates a new apiServerMonitor, panics or exits if an error occurs.
+func NewAPIServerMonitorOrDie(configPath string) types.Monitor {
+	a := &apiServerMonitor{
+		configPath: configPath,
+		tomb:       tomb.NewTomb(),
+	}
+
+	f, err := os.ReadFile(configPath)
+	if err != nil {
+		klog.Fatalf("Failed to read configuration file %q: %v", configPath, err)
+	}
+	if err := json.Unmarshal(f, &a.config); err != nil {
+		klog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err)
+	}
+	if err := a.config.ApplyConfiguration(); err != nil {
+		klog.Fatalf("Failed to apply configuration for %q: %v", configPath, err)
+	}
+	if err := a.config.Validate(); err != nil {
+		klog.Fatalf("Failed to validate apiserver monitor config %+v: %v", a.config, err)
+	}
+
+	klog.Infof("Finish parsing apiserver monitor config file %s: %+v", a.configPath, a.config)
+
+	if util.GlobalKubeClientConfig == nil {
+		klog.Fatalf("GlobalKubeClientConfig is not initialized")
+	}
+	kubeConfig := kube_rest.CopyConfig(util.GlobalKubeClientConfig)
+	kubeConfig.Timeout = a.config.Timeout
+	if kubeConfig.Proxy == nil {
+		// Explicitly set Proxy so client-go's tlsTransportCache treats this config as
+		// uncacheable and creates a fresh http.Transport on each RESTClientFor call
+		// instead of reusing a cached transport and its already-established TCP connection.
+		kubeConfig.Proxy = http.ProxyFromEnvironment
+	}
+	a.kubeConfig = kubeConfig
+
+	// A 1000 size channel should be big enough.
+	a.statusChan = make(chan *types.Status, 1000)
+
+	if *a.config.EnableMetricsReporting {
+		initializeProblemMetricsOrDie(a.config.DefaultConditions)
+	}
+	return a
+}
+
+// initializeProblemMetricsOrDie creates problem metrics for all configured conditions and sets the initial value to 0.
+func initializeProblemMetricsOrDie(conditions []types.Condition) {
+	if problemmetrics.GlobalProblemMetricsManager == nil {
+		return
+	}
+	for _, cond := range conditions {
+		reason := cond.Type
+		if err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(cond.Type, reason, false); err != nil {
+			klog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
+				cond.Type, reason, err)
+		}
+		if err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(reason, 0); err != nil {
+			klog.Fatalf("Failed to initialize problem counter metrics for %q: %v", reason, err)
+		}
+	}
+}
+
+func (a *apiServerMonitor) Start() (<-chan *types.Status, error) {
+	klog.Infof("Start apiserver monitor %s", a.configPath)
+	go a.monitorLoop()
+	return a.statusChan, nil
+}
+
+func (a *apiServerMonitor) Stop() {
+	klog.Infof("Stop apiserver monitor %s", a.configPath)
+	a.tomb.Stop()
+}
+
+func (a *apiServerMonitor) monitorLoop() {
+	defer a.tomb.Done()
+
+	a.initializeConditions()
+	if *a.config.SkipInitialStatus {
+		klog.Infof("Skipping sending initial status. Using default conditions: %+v", a.conditions)
+	} else {
+		a.sendInitialStatus()
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go func() {
+		select {
+		case <-a.tomb.Stopping():
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+
+	runTicker := time.NewTicker(a.config.InvokeInterval)
+	defer runTicker.Stop()
+
+	select {
+	case <-a.tomb.Stopping():
+		klog.Infof("APIServer monitor stopped: %s", a.configPath)
+		return
+	default:
+		a.runCheck(ctx)
+	}
+
+	for {
+		select {
+		case <-runTicker.C:
+			a.runCheck(ctx)
+		case <-a.tomb.Stopping():
+			klog.Infof("APIServer monitor stopped: %s", a.configPath)
+			return
+		}
+	}
+}
+
+func (a *apiServerMonitor) runCheck(ctx context.Context) {
+	err := a.checkAPIServer(ctx)
+	select {
+	case <-a.tomb.Stopping():
+		return
+	default:
+	}
+	status := a.generateStatus(err)
+	klog.V(3).Infof("New status generated: %+v", status)
+	a.statusChan <- status
+}
+
+func (a *apiServerMonitor) checkAPIServer(ctx context.Context) error {
+	checkCtx, cancel := context.WithTimeout(ctx, a.config.Timeout)
+	defer cancel()
+
+	client, err := kube_rest.RESTClientFor(a.kubeConfig)
+	if err != nil {
+		return fmt.Errorf("failed to create REST client: %w", err)
+	}
+	if client.Client != nil {
+		defer utilnet.CloseIdleConnectionsFor(client.Client.Transport)
+	}
+	return client.Get().AbsPath("/version").Do(checkCtx).Error()
+}
+
+func (a *apiServerMonitor) generateStatus(checkErr error) *types.Status {
+	timestamp := time.Now()
+	var activeProblemEvents []types.Event
+	var inactiveProblemEvents []types.Event
+
+	for i := range a.conditions {
+		condition := &a.conditions[i]
+		defaultCondition := a.config.DefaultConditions[i]
+
+		newStatus := types.False
+		newReason := defaultCondition.Reason
+		newMessage := defaultCondition.Message
+		if checkErr != nil {
+			newStatus = types.True
+			newReason = condition.Type
+			newMessage = checkErr.Error()
+		}
+
+		if condition.Status != newStatus {
+			condition.Transition = timestamp
+			condition.Status = newStatus
+			condition.Reason = newReason
+			condition.Message = newMessage
+
+			updateEvent := util.GenerateConditionChangeEvent(
+				condition.Type,
+				newStatus,
+				newReason,
+				newMessage,
+				timestamp,
+			)
+			if newStatus == types.True {
+				activeProblemEvents = append(activeProblemEvents, updateEvent)
+			} else {
+				inactiveProblemEvents = append(inactiveProblemEvents, updateEvent)
+			}
+		}
+	}
+
+	if *a.config.EnableMetricsReporting && problemmetrics.GlobalProblemMetricsManager != nil {
+		for _, event := range activeProblemEvents {
+			if err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(event.Reason, 1); err != nil {
+				klog.Errorf("Failed to update problem counter metrics for %q: %v", event.Reason, err)
+			}
+		}
+		for _, condition := range a.conditions {
+			if err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(
+				condition.Type, condition.Reason, condition.Status == types.True); err != nil {
+				klog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+					condition.Type, condition.Reason, err)
+			}
+		}
+	}
+
+	status := &types.Status{
+		Source:     a.config.Source,
+		Events:     append(activeProblemEvents, inactiveProblemEvents...),
+		Conditions: slices.Clone(a.conditions),
+	}
+	if len(activeProblemEvents) != 0 || len(inactiveProblemEvents) != 0 {
+		klog.V(0).Infof("New status generated: %+v", status)
+	}
+	return status
+}
+
+func (a *apiServerMonitor) sendInitialStatus() {
+	klog.Infof("Sending initial status for %s with conditions: %+v", a.config.Source, a.conditions)
+	a.statusChan <- &types.Status{
+		Source:     a.config.Source,
+		Conditions: slices.Clone(a.conditions),
+	}
+}
+
+func (a *apiServerMonitor) initializeConditions() {
+	a.conditions = initialConditions(a.config.DefaultConditions)
+	klog.Infof("Initialized conditions for %s: %+v", a.configPath, a.conditions)
+}
+
+func initialConditions(defaults []types.Condition) []types.Condition {
+	conditions := make([]types.Condition, len(defaults))
+	copy(conditions, defaults)
+	for i := range conditions {
+		conditions[i].Status = types.False
+		conditions[i].Transition = time.Now()
+	}
+	return conditions
+}