Skip to content

Commit f192657

Browse files
committed
Add gauge metric for master of leader election.
Fixes kubernetes#71730 0 indicates standby, 1 indicates master, label indicates which lease. Tweaked name and documentation Factored in Mike Danese feedback. Removed dependency on prometheus from client-go using adapter. Centralized adapter import. Fixed godeps Fixed boilerplate. Put in fixes for caesarxuchao
1 parent 68451f3 commit f192657

File tree

20 files changed

+271
-29
lines changed

20 files changed

+271
-29
lines changed

cmd/cloud-controller-manager/BUILD

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ go_library(
2020
importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
2121
deps = [
2222
"//cmd/cloud-controller-manager/app:go_default_library",
23-
"//pkg/client/metrics/prometheus:go_default_library",
2423
"//pkg/cloudprovider/providers:go_default_library",
24+
"//pkg/util/prometheusclientgo:go_default_library",
2525
"//pkg/version/prometheus:go_default_library",
2626
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
2727
],

cmd/cloud-controller-manager/controller-manager.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ import (
2727

2828
"k8s.io/apiserver/pkg/util/logs"
2929
"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
30-
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
3130
// NOTE: Importing all in-tree cloud-providers is not required when
3231
// implementing an out-of-tree cloud-provider.
3332
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
34-
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
33+
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
34+
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
3535
)
3636

3737
func main() {

cmd/kube-apiserver/BUILD

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ go_library(
2020
importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
2121
deps = [
2222
"//cmd/kube-apiserver/app:go_default_library",
23-
"//pkg/client/metrics/prometheus:go_default_library",
23+
"//pkg/util/prometheusclientgo:go_default_library",
2424
"//pkg/version/prometheus:go_default_library",
2525
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
2626
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

cmd/kube-apiserver/apiserver.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ import (
2727
"k8s.io/apiserver/pkg/server"
2828
"k8s.io/apiserver/pkg/util/logs"
2929
"k8s.io/kubernetes/cmd/kube-apiserver/app"
30-
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
31-
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
30+
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
31+
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
3232
)
3333

3434
func main() {

cmd/kube-controller-manager/BUILD

+1-3
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ go_library(
2020
importpath = "k8s.io/kubernetes/cmd/kube-controller-manager",
2121
deps = [
2222
"//cmd/kube-controller-manager/app:go_default_library",
23-
"//pkg/client/metrics/prometheus:go_default_library",
24-
"//pkg/util/reflector/prometheus:go_default_library",
25-
"//pkg/util/workqueue/prometheus:go_default_library",
23+
"//pkg/util/prometheusclientgo:go_default_library",
2624
"//pkg/version/prometheus:go_default_library",
2725
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
2826
],

cmd/kube-controller-manager/controller-manager.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,8 @@ import (
2828

2929
"k8s.io/apiserver/pkg/util/logs"
3030
"k8s.io/kubernetes/cmd/kube-controller-manager/app"
31-
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
32-
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
33-
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
34-
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
31+
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugin
32+
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
3533
)
3634

3735
func main() {

cmd/kube-scheduler/BUILD

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ go_library(
2020
importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
2121
deps = [
2222
"//cmd/kube-scheduler/app:go_default_library",
23-
"//pkg/client/metrics/prometheus:go_default_library",
23+
"//pkg/util/prometheusclientgo:go_default_library",
2424
"//pkg/version/prometheus:go_default_library",
2525
"//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
2626
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

cmd/kube-scheduler/scheduler.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ import (
2727
utilflag "k8s.io/apiserver/pkg/util/flag"
2828
"k8s.io/apiserver/pkg/util/logs"
2929
"k8s.io/kubernetes/cmd/kube-scheduler/app"
30-
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
31-
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
30+
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
31+
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
3232
)
3333

3434
func main() {

pkg/controller/garbagecollector/BUILD

-2
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ go_library(
2222
deps = [
2323
"//pkg/controller:go_default_library",
2424
"//pkg/controller/garbagecollector/metaonly:go_default_library",
25-
"//pkg/util/reflector/prometheus:go_default_library",
26-
"//pkg/util/workqueue/prometheus:go_default_library",
2725
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
2826
"//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
2927
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/controller/garbagecollector/garbagecollector.go

-3
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,6 @@ import (
3939
"k8s.io/client-go/informers"
4040
"k8s.io/client-go/util/workqueue"
4141
"k8s.io/kubernetes/pkg/controller"
42-
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
43-
// install the prometheus plugin
44-
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
4542
// import known versions
4643
_ "k8s.io/client-go/kubernetes"
4744
)

pkg/util/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ filegroup(
4747
"//pkg/util/parsers:all-srcs",
4848
"//pkg/util/pod:all-srcs",
4949
"//pkg/util/procfs:all-srcs",
50+
"//pkg/util/prometheusclientgo:all-srcs",
5051
"//pkg/util/reflector/prometheus:all-srcs",
5152
"//pkg/util/removeall:all-srcs",
5253
"//pkg/util/resizefs:all-srcs",

pkg/util/prometheusclientgo/BUILD

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["adapters.go"],
6+
importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//pkg/client/metrics/prometheus:go_default_library",
10+
"//pkg/util/prometheusclientgo/leaderelection:go_default_library",
11+
"//pkg/util/reflector/prometheus:go_default_library",
12+
"//pkg/util/workqueue/prometheus:go_default_library",
13+
],
14+
)
15+
16+
filegroup(
17+
name = "package-srcs",
18+
srcs = glob(["**"]),
19+
tags = ["automanaged"],
20+
visibility = ["//visibility:private"],
21+
)
22+
23+
filegroup(
24+
name = "all-srcs",
25+
srcs = [
26+
":package-srcs",
27+
"//pkg/util/prometheusclientgo/leaderelection:all-srcs",
28+
],
29+
tags = ["automanaged"],
30+
visibility = ["//visibility:public"],
31+
)
+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package prometheusclientgo
18+
19+
// Provided metrics needing adapting
20+
import (
21+
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
22+
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection" // for leader election metric registration
23+
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
24+
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
25+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["adapter.go"],
6+
importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
10+
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
11+
],
12+
)
13+
14+
filegroup(
15+
name = "package-srcs",
16+
srcs = glob(["**"]),
17+
tags = ["automanaged"],
18+
visibility = ["//visibility:private"],
19+
)
20+
21+
filegroup(
22+
name = "all-srcs",
23+
srcs = [":package-srcs"],
24+
tags = ["automanaged"],
25+
visibility = ["//visibility:public"],
26+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package prometheus
18+
19+
import (
20+
"github.com/prometheus/client_golang/prometheus"
21+
"k8s.io/client-go/tools/leaderelection"
22+
)
23+
24+
// Package prometheus sets the workqueue DefaultMetricsFactory to produce
25+
// prometheus metrics. To use this package, you just have to import it.
26+
27+
func init() {
28+
leaderelection.SetProvider(prometheusMetricsProvider{})
29+
}
30+
31+
type prometheusMetricsProvider struct{}
32+
33+
func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric {
34+
leaderGauge := prometheus.NewGaugeVec(
35+
prometheus.GaugeOpts{
36+
Name: "leader_election_master_status",
37+
Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
38+
},
39+
[]string{"name"},
40+
)
41+
prometheus.Register(leaderGauge)
42+
return &switchAdapter{gauge: leaderGauge}
43+
}
44+
45+
type switchAdapter struct {
46+
gauge *prometheus.GaugeVec
47+
}
48+
49+
func (s *switchAdapter) On(name string) {
50+
s.gauge.WithLabelValues(name).Set(1.0)
51+
}
52+
53+
func (s *switchAdapter) Off(name string) {
54+
s.gauge.WithLabelValues(name).Set(0.0)
55+
}

plugin/pkg/admission/resourcequota/BUILD

-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ go_library(
2020
"//pkg/kubeapiserver/admission:go_default_library",
2121
"//pkg/quota/v1:go_default_library",
2222
"//pkg/quota/v1/generic:go_default_library",
23-
"//pkg/util/reflector/prometheus:go_default_library",
24-
"//pkg/util/workqueue/prometheus:go_default_library",
2523
"//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library",
2624
"//plugin/pkg/admission/resourcequota/apis/resourcequota/install:go_default_library",
2725
"//plugin/pkg/admission/resourcequota/apis/resourcequota/v1beta1:go_default_library",

plugin/pkg/admission/resourcequota/controller.go

-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ import (
3737
"k8s.io/client-go/util/workqueue"
3838
quota "k8s.io/kubernetes/pkg/quota/v1"
3939
"k8s.io/kubernetes/pkg/quota/v1/generic"
40-
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
41-
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
4240
resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota"
4341
)
4442

staging/src/k8s.io/client-go/tools/leaderelection/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ go_library(
1111
srcs = [
1212
"healthzadaptor.go",
1313
"leaderelection.go",
14+
"metrics.go",
1415
],
1516
importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection",
1617
importpath = "k8s.io/client-go/tools/leaderelection",

staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,13 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
8989
if lec.Lock == nil {
9090
return nil, fmt.Errorf("Lock must not be nil.")
9191
}
92-
return &LeaderElector{
93-
config: lec,
94-
clock: clock.RealClock{},
95-
}, nil
92+
le := LeaderElector{
93+
config: lec,
94+
clock: clock.RealClock{},
95+
metrics: globalMetricsFactory.newLeaderMetrics(),
96+
}
97+
le.metrics.leaderOff(le.config.Name)
98+
return &le, nil
9699
}
97100

98101
type LeaderElectionConfig struct {
@@ -152,6 +155,8 @@ type LeaderElector struct {
152155
// clock is wrapper around time to allow for less flaky testing
153156
clock clock.Clock
154157

158+
metrics leaderMetricsAdapter
159+
155160
// name is the name of the resource lock for debugging
156161
name string
157162
}
@@ -211,6 +216,7 @@ func (le *LeaderElector) acquire(ctx context.Context) bool {
211216
return
212217
}
213218
le.config.Lock.RecordEvent("became leader")
219+
le.metrics.leaderOn(le.config.Name)
214220
klog.Infof("successfully acquired lease %v", desc)
215221
cancel()
216222
}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
@@ -246,6 +252,7 @@ func (le *LeaderElector) renew(ctx context.Context) {
246252
return
247253
}
248254
le.config.Lock.RecordEvent("stopped leading")
255+
le.metrics.leaderOff(le.config.Name)
249256
klog.Infof("failed to renew lease %v: %v", desc, err)
250257
cancel()
251258
}, le.config.RetryPeriod, ctx.Done())

0 commit comments

Comments
 (0)