Skip to content

Commit ad78f73

Browse files
ywelschywelsch
authored
Add term and config to cluster state (#32100)
Adds the publication term and the last accepted and committed configurations to the cluster state, following the formal model in https://github.com/elastic/elasticsearch-formal-models/blob/master/ZenWithTerms/tla/ZenWithTerms.tla The term represents the reign of a master, and the last committed / accepted configurations represent the set of quorums that cluster state changes will require (If there's no reconfiguration, last accepted and last committed configurations coincide).
1 parent e31a877 commit ad78f73

File tree

8 files changed

+283
-11
lines changed

8 files changed

+283
-11
lines changed

‎build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -173,8 +173,8 @@ task verifyVersions {
173173
* the enabled state of every bwc task. It should be set back to true
174174
* after the backport of the backcompat code is complete.
175175
*/
176-
final boolean bwc_tests_enabled = true
177-
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
176+
final boolean bwc_tests_enabled = false
177+
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/issues/32006" // BWC handled at a later time
178178
if (bwc_tests_enabled == false) {
179179
if (bwc_tests_disabled_issue.isEmpty()) {
180180
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

‎server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

+3
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,11 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS
7676
ClusterState currentState = clusterService.state();
7777
logger.trace("Serving cluster state request using version {}", currentState.version());
7878
ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
79+
builder.term(currentState.term());
7980
builder.version(currentState.version());
8081
builder.stateUUID(currentState.stateUUID());
82+
builder.lastCommittedConfiguration(currentState.getLastCommittedConfiguration());
83+
builder.lastAcceptedConfiguration(currentState.getLastAcceptedConfiguration());
8184
if (request.nodes()) {
8285
builder.nodes(currentState.nodes());
8386
}

‎server/src/main/java/org/elasticsearch/cluster/ClusterState.java

+173-6
Large diffs are not rendered by default.

‎server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java

+6
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,11 @@ public void testToXContent() throws IOException {
7070
" \"acknowledged\" : true,\n" +
7171
" \"state\" : {\n" +
7272
" \"cluster_uuid\" : \"_na_\",\n" +
73+
" \"term\" : 0,\n" +
7374
" \"version\" : 0,\n" +
7475
" \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" +
76+
" \"last_committed_config\" : [ ],\n" +
77+
" \"last_accepted_config\" : [ ],\n" +
7578
" \"master_node\" : \"node0\",\n" +
7679
" \"blocks\" : { },\n" +
7780
" \"nodes\" : {\n" +
@@ -138,8 +141,11 @@ public void testToXContent() throws IOException {
138141
" \"acknowledged\" : true,\n" +
139142
" \"state\" : {\n" +
140143
" \"cluster_uuid\" : \"_na_\",\n" +
144+
" \"term\" : 0,\n" +
141145
" \"version\" : 0,\n" +
142146
" \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" +
147+
" \"last_committed_config\" : [ ],\n" +
148+
" \"last_accepted_config\" : [ ],\n" +
143149
" \"master_node\" : \"node0\"\n" +
144150
" },\n" +
145151
" \"explanations\" : [\n" +

‎server/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

+22-2
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public void testClusterStateDiffSerialization() throws Exception {
9797
if (i > 0) {
9898
clusterState = builder.build();
9999
}
100-
switch (randomInt(4)) {
100+
switch (randomInt(5)) {
101101
case 0:
102102
builder = randomNodes(clusterState);
103103
break;
@@ -113,11 +113,14 @@ public void testClusterStateDiffSerialization() throws Exception {
113113
case 4:
114114
builder = randomMetaDataChanges(clusterState);
115115
break;
116+
case 5:
117+
builder = randomVotingConfiguration(clusterState);
118+
break;
116119
default:
117120
throw new IllegalArgumentException("Shouldn't be here");
118121
}
119122
}
120-
clusterState = builder.incrementVersion().build();
123+
clusterState = builder.incrementVersion().term(randomLong()).build();
121124

122125
if (randomIntBetween(0, 10) < 1) {
123126
// Update cluster state via full serialization from time to time
@@ -141,7 +144,10 @@ public void testClusterStateDiffSerialization() throws Exception {
141144
try {
142145
// Check non-diffable elements
143146
assertThat(clusterStateFromDiffs.version(), equalTo(clusterState.version()));
147+
assertThat(clusterStateFromDiffs.term(), equalTo(clusterState.term()));
144148
assertThat(clusterStateFromDiffs.stateUUID(), equalTo(clusterState.stateUUID()));
149+
assertThat(clusterStateFromDiffs.getLastAcceptedConfiguration(), equalTo(clusterState.getLastAcceptedConfiguration()));
150+
assertThat(clusterStateFromDiffs.getLastCommittedConfiguration(), equalTo(clusterState.getLastCommittedConfiguration()));
145151

146152
// Check nodes
147153
assertThat(clusterStateFromDiffs.nodes().getNodes(), equalTo(clusterState.nodes().getNodes()));
@@ -190,6 +196,20 @@ public void testClusterStateDiffSerialization() throws Exception {
190196

191197
}
192198

199+
private ClusterState.Builder randomVotingConfiguration(ClusterState clusterState) {
200+
ClusterState.Builder builder = ClusterState.builder(clusterState);
201+
if (randomBoolean()) {
202+
builder.lastCommittedConfiguration(
203+
new ClusterState.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(10, 10, false))));
204+
}
205+
if (randomBoolean()) {
206+
builder.lastAcceptedConfiguration(
207+
new ClusterState.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(10, 10, false))));
208+
}
209+
210+
return builder;
211+
}
212+
193213
/**
194214
* Randomly updates nodes in the cluster state
195215
*/

‎server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java

+72
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@
1919
package org.elasticsearch.cluster;
2020

2121
import org.elasticsearch.Version;
22+
import org.elasticsearch.cluster.ClusterState.VotingConfiguration;
2223
import org.elasticsearch.cluster.node.DiscoveryNode;
2324
import org.elasticsearch.cluster.node.DiscoveryNodes;
25+
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
2426
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.util.set.Sets;
2528
import org.elasticsearch.test.ESTestCase;
29+
import org.elasticsearch.test.EqualsHashCodeTestUtils;
30+
31+
import java.util.Collections;
32+
import java.util.HashSet;
33+
import java.util.Set;
2634

2735
import static java.util.Collections.emptyMap;
2836
import static java.util.Collections.emptySet;
@@ -55,6 +63,70 @@ public void testSupersedes() {
5563

5664
// state from the same master compare by version
5765
assertThat(withMaster1a.supersedes(withMaster1b), equalTo(withMaster1a.version() > withMaster1b.version()));
66+
}
67+
68+
public void testVotingConfiguration() {
69+
VotingConfiguration config0 = new VotingConfiguration(Sets.newHashSet());
70+
assertThat(config0, equalTo(VotingConfiguration.EMPTY_CONFIG));
71+
assertThat(config0.getNodeIds(), equalTo(Sets.newHashSet()));
72+
assertThat(config0.isEmpty(), equalTo(true));
73+
assertThat(config0.hasQuorum(Sets.newHashSet()), equalTo(false));
74+
assertThat(config0.hasQuorum(Sets.newHashSet("id1")), equalTo(false));
75+
76+
VotingConfiguration config1 = new VotingConfiguration(Sets.newHashSet("id1"));
77+
assertThat(config1.getNodeIds(), equalTo(Sets.newHashSet("id1")));
78+
assertThat(config1.isEmpty(), equalTo(false));
79+
assertThat(config1.hasQuorum(Sets.newHashSet("id1")), equalTo(true));
80+
assertThat(config1.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
81+
assertThat(config1.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
82+
assertThat(config1.hasQuorum(Sets.newHashSet()), equalTo(false));
83+
84+
VotingConfiguration config2 = new VotingConfiguration(Sets.newHashSet("id1", "id2"));
85+
assertThat(config2.getNodeIds(), equalTo(Sets.newHashSet("id1", "id2")));
86+
assertThat(config2.isEmpty(), equalTo(false));
87+
assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
88+
assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id2", "id3")), equalTo(true));
89+
assertThat(config2.hasQuorum(Sets.newHashSet("id1")), equalTo(false));
90+
assertThat(config2.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
91+
assertThat(config2.hasQuorum(Sets.newHashSet("id3")), equalTo(false));
92+
assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id3")), equalTo(false));
93+
assertThat(config2.hasQuorum(Sets.newHashSet()), equalTo(false));
94+
95+
VotingConfiguration config3 = new VotingConfiguration(Sets.newHashSet("id1", "id2", "id3"));
96+
assertThat(config3.getNodeIds(), equalTo(Sets.newHashSet("id1", "id2", "id3")));
97+
assertThat(config3.isEmpty(), equalTo(false));
98+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
99+
assertThat(config3.hasQuorum(Sets.newHashSet("id2", "id3")), equalTo(true));
100+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id3")), equalTo(true));
101+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2", "id3")), equalTo(true));
102+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2", "id4")), equalTo(true));
103+
assertThat(config3.hasQuorum(Sets.newHashSet("id1")), equalTo(false));
104+
assertThat(config3.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
105+
assertThat(config3.hasQuorum(Sets.newHashSet("id3")), equalTo(false));
106+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id4")), equalTo(false));
107+
assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id4", "id5")), equalTo(false));
108+
assertThat(config3.hasQuorum(Sets.newHashSet()), equalTo(false));
109+
}
58110

111+
public void testVotingConfigurationSerializationEqualsHashCode() {
112+
VotingConfiguration initialConfig = new VotingConfiguration(
113+
Sets.newHashSet(generateRandomStringArray(randomInt(10), 20, false)));
114+
EqualsHashCodeTestUtils.checkEqualsAndHashCode(initialConfig,
115+
orig -> ESTestCase.copyWriteable(orig, new NamedWriteableRegistry(Collections.emptyList()), VotingConfiguration::new),
116+
cfg -> {
117+
Set<String> newNodeIds = new HashSet<>(cfg.getNodeIds());
118+
if (cfg.isEmpty() == false && randomBoolean()) {
119+
// remove random element
120+
newNodeIds.remove(randomFrom(cfg.getNodeIds()));
121+
} else if (cfg.isEmpty() == false && randomBoolean()) {
122+
// change random element
123+
newNodeIds.remove(randomFrom(cfg.getNodeIds()));
124+
newNodeIds.add(randomAlphaOfLength(20));
125+
} else {
126+
// add random element
127+
newNodeIds.add(randomAlphaOfLength(20));
128+
}
129+
return new VotingConfiguration(newNodeIds);
130+
});
59131
}
60132
}

‎server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,8 @@ public void clusterChanged(ClusterChangedEvent event) {
487487
clusterState = ClusterState.builder(clusterState).blocks(ClusterBlocks.builder()
488488
.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).incrementVersion().build();
489489

490-
ClusterState unserializableClusterState = new ClusterState(clusterState.version(), clusterState.stateUUID(), clusterState) {
490+
ClusterState unserializableClusterState = new ClusterState(clusterState.term(), clusterState.version(), clusterState.stateUUID(),
491+
clusterState) {
491492
@Override
492493
public Diff<ClusterState> diff(ClusterState previousState) {
493494
return new Diff<ClusterState>() {

‎x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -504,8 +504,11 @@ public void testToXContent() throws IOException {
504504
+ "\"nodes_hash\":1314980060,"
505505
+ "\"status\":\"green\","
506506
+ "\"cluster_uuid\":\"_cluster\","
507+
+ "\"term\":0,"
507508
+ "\"version\":12,"
508509
+ "\"state_uuid\":\"_state_uuid\","
510+
+ "\"last_committed_config\":[],"
511+
+ "\"last_accepted_config\":[],"
509512
+ "\"master_node\":\"_node\","
510513
+ "\"nodes\":{"
511514
+ "\"_node_id\":{"

0 commit comments

Comments
 (0)