iterator = Splitter.on(File.separatorChar).split(path).iterator();
+ boolean found = false;
+ File dir = dstDir;
+ while (iterator.hasNext()) {
+ String current = iterator.next();
+ if (!found && flag.equals(current)) {
+ found = true;
+ if (into) {
+ dir = new File(dir, flag);
+ if (!dir.exists()) {
+ Assert.assertTrue(dir.mkdirs());
+ }
+ }
+ continue;
+ }
+
+ if (found) {
+ if (!iterator.hasNext()) {
+ dir = new File(dir, current);
+ } else {
+ dir = new File(dir, current);
+ if (!dir.exists()) {
+ Assert.assertTrue(dir.mkdir());
+ }
+ }
+ }
+ }
+
+ Assert.assertTrue(dir.createNewFile());
+ byte[] buffer = new byte[4096];
+ BufferedInputStream bis = new BufferedInputStream(src);
+ int len = 0;
+ try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(dir.toPath()))) {
+ while ((len = bis.read(buffer)) > 0) {
+ bos.write(buffer, 0, len);
+ }
+ }
+ }
+
+ public static void recursiveDelete(File file) {
+ if (file.isFile()) {
+ file.delete();
+ } else {
+ File[] files = file.listFiles();
+ if (null != files) {
+ for (File f : files) {
+ recursiveDelete(f);
+ }
+ }
+ file.delete();
+ }
+ }
+
+ public static File copyResources(String folder) throws IOException {
+ return copyResources(folder, false);
+ }
+
+ public static File copyResources(String folder, boolean into) throws IOException {
+ File home = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString().replace('-', '_'));
+ if (!home.exists()) {
+ Assert.assertTrue(home.mkdirs());
+ }
+ PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(AclTestHelper.class.getClassLoader());
+ Resource[] resources = resolver.getResources(String.format("classpath:%s/**/*", folder));
+ for (Resource resource : resources) {
+ if (!resource.isReadable()) {
+ continue;
+ }
+ String description = resource.getDescription();
+ int start = description.indexOf('[');
+ int end = description.lastIndexOf(']');
+ String path = description.substring(start + 1, end);
+ try (InputStream inputStream = resource.getInputStream()) {
+ copyTo(path, inputStream, home, folder, into);
+ }
+ }
+ return home;
+ }
+}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
new file mode 100644
index 00000000000..5193457146d
--- /dev/null
+++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.acl.plain;
+
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.AclConstants;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.common.AclConfig;
+import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * In this class, we'll test the following scenarios, each containing several consecutive operations on ACL,
+ *
like updating and deleting ACL, changing config files and checking validations.
+ *
Case 1: Only conf/plain_acl.yml exists;
+ *
Case 2: Only conf/acl/plain_acl.yml exists;
+ *
Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists.
+ */
+public class PlainAccessControlFlowTest {
+ public static final String DEFAULT_TOPIC = "topic-acl";
+
+ public static final String DEFAULT_GROUP = "GID_acl";
+
+ public static final String DEFAULT_PRODUCER_AK = "ak11111";
+ public static final String DEFAULT_PRODUCER_SK = "1234567";
+
+ public static final String DEFAULT_CONSUMER_SK = "7654321";
+ public static final String DEFAULT_CONSUMER_AK = "ak22222";
+
+ public static final String DEFAULT_GLOBAL_WHITE_ADDR = "172.16.123.123";
+ public static final List DEFAULT_GLOBAL_WHITE_ADDRS_LIST = Collections.singletonList(DEFAULT_GLOBAL_WHITE_ADDR);
+
+ @Test
+ public void testEmptyAclFolderCase() throws NoSuchFieldException, IllegalAccessException,
+ IOException {
+ String folder = "empty_acl_folder_conf";
+ File home = AclTestHelper.copyResources(folder);
+ System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ checkDefaultAclFileExists();
+ testValidationAfterConsecutiveUpdates(plainAccessValidator);
+ testValidationAfterConfigFileChanged(plainAccessValidator);
+ AclTestHelper.recursiveDelete(home);
+ }
+
+ @Test
+ public void testOnlyAclFolderCase() throws NoSuchFieldException, IllegalAccessException, IOException {
+ String folder = "only_acl_folder_conf";
+ File home = AclTestHelper.copyResources(folder);
+ System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ checkDefaultAclFileExists();
+ testValidationAfterConsecutiveUpdates(plainAccessValidator);
+ testValidationAfterConfigFileChanged(plainAccessValidator);
+ AclTestHelper.recursiveDelete(home);
+ }
+
+ @Test
+ public void testBothAclFileAndFolderCase() throws NoSuchFieldException, IllegalAccessException,
+ IOException {
+ String folder = "both_acl_file_folder_conf";
+ File root = AclTestHelper.copyResources(folder);
+ System.setProperty("rocketmq.home.dir", root.getAbsolutePath());
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ checkDefaultAclFileExists();
+ testValidationAfterConsecutiveUpdates(plainAccessValidator);
+ testValidationAfterConfigFileChanged(plainAccessValidator);
+ AclTestHelper.recursiveDelete(root);
+ }
+
+ private void testValidationAfterConfigFileChanged(
+ PlainAccessValidator plainAccessValidator) throws NoSuchFieldException, IllegalAccessException {
+ PlainAccessConfig producerAccessConfig = generateProducerAccessConfig();
+ PlainAccessConfig consumerAccessConfig = generateConsumerAccessConfig();
+ List plainAccessConfigList = new LinkedList<>();
+ plainAccessConfigList.add(producerAccessConfig);
+ plainAccessConfigList.add(consumerAccessConfig);
+ PlainAccessData ymlMap = new PlainAccessData();
+ ymlMap.setAccounts(plainAccessConfigList);
+
+ // write prepared PlainAccessConfigs to file
+ final String aclConfigFile = System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml";
+ AclUtils.writeDataObject(aclConfigFile, ymlMap);
+
+ loadConfigFile(plainAccessValidator, aclConfigFile);
+
+ // check if added successfully
+ final AclConfig allAclConfig = plainAccessValidator.getAllAclConfig();
+ final List plainAccessConfigs = allAclConfig.getPlainAccessConfigs();
+ checkPlainAccessConfig(producerAccessConfig, plainAccessConfigs);
+ checkPlainAccessConfig(consumerAccessConfig, plainAccessConfigs);
+
+ //delete consumer account
+ plainAccessConfigList.remove(consumerAccessConfig);
+ AclUtils.writeDataObject(aclConfigFile, ymlMap);
+
+ loadConfigFile(plainAccessValidator, aclConfigFile);
+
+ // sending messages will be successful using prepared credentials
+ SessionCredentials producerCredential = new SessionCredentials(DEFAULT_PRODUCER_AK, DEFAULT_PRODUCER_SK);
+ AclClientRPCHook producerHook = new AclClientRPCHook(producerCredential);
+ validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+ validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+
+ // consuming messages will be failed for account has been deleted
+ SessionCredentials consumerCredential = new SessionCredentials(DEFAULT_CONSUMER_AK, DEFAULT_CONSUMER_SK);
+ AclClientRPCHook consumerHook = new AclClientRPCHook(consumerCredential);
+ boolean isConsumeFailed = false;
+ try {
+ validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
+ } catch (AclException e) {
+ isConsumeFailed = true;
+ }
+ Assert.assertTrue("Message should not be consumed after account deleted", isConsumeFailed);
+
+ }
+
+ private void testValidationAfterConsecutiveUpdates(
+ PlainAccessValidator plainAccessValidator) throws NoSuchFieldException, IllegalAccessException {
+ PlainAccessConfig producerAccessConfig = generateProducerAccessConfig();
+ plainAccessValidator.updateAccessConfig(producerAccessConfig);
+
+ PlainAccessConfig consumerAccessConfig = generateConsumerAccessConfig();
+ plainAccessValidator.updateAccessConfig(consumerAccessConfig);
+
+ plainAccessValidator.updateGlobalWhiteAddrsConfig(DEFAULT_GLOBAL_WHITE_ADDRS_LIST, null);
+
+ // check if the above config updated successfully
+ final AclConfig allAclConfig = plainAccessValidator.getAllAclConfig();
+ final List plainAccessConfigs = allAclConfig.getPlainAccessConfigs();
+ checkPlainAccessConfig(producerAccessConfig, plainAccessConfigs);
+ checkPlainAccessConfig(consumerAccessConfig, plainAccessConfigs);
+
+ Assert.assertEquals(DEFAULT_GLOBAL_WHITE_ADDRS_LIST, allAclConfig.getGlobalWhiteAddrs());
+
+ // check sending and consuming messages
+ SessionCredentials producerCredential = new SessionCredentials(DEFAULT_PRODUCER_AK, DEFAULT_PRODUCER_SK);
+ AclClientRPCHook producerHook = new AclClientRPCHook(producerCredential);
+ validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+ validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+
+ SessionCredentials consumerCredential = new SessionCredentials(DEFAULT_CONSUMER_AK, DEFAULT_CONSUMER_SK);
+ AclClientRPCHook consumerHook = new AclClientRPCHook(consumerCredential);
+ validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
+
+ // load from file
+ loadConfigFile(plainAccessValidator,
+ System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml");
+ SessionCredentials unmatchedCredential = new SessionCredentials("non_exists_sk", "non_exists_sk");
+ AclClientRPCHook dummyHook = new AclClientRPCHook(unmatchedCredential);
+ validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
+ validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
+ validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
+
+ //recheck after reloading
+ validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+ validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
+ validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
+
+ }
+
+ private void loadConfigFile(PlainAccessValidator plainAccessValidator,
+ String configFileName) throws NoSuchFieldException, IllegalAccessException {
+ Class clazz = PlainAccessValidator.class;
+ Field f = clazz.getDeclaredField("aclPlugEngine");
+ f.setAccessible(true);
+ PlainPermissionManager aclPlugEngine = (PlainPermissionManager) f.get(plainAccessValidator);
+ aclPlugEngine.load(configFileName);
+ }
+
+ private PlainAccessConfig generateConsumerAccessConfig() {
+ PlainAccessConfig plainAccessConfig2 = new PlainAccessConfig();
+ plainAccessConfig2.setAccessKey(DEFAULT_CONSUMER_AK);
+ plainAccessConfig2.setSecretKey(DEFAULT_CONSUMER_SK);
+ plainAccessConfig2.setAdmin(false);
+ plainAccessConfig2.setDefaultTopicPerm(AclConstants.DENY);
+ plainAccessConfig2.setDefaultGroupPerm(AclConstants.DENY);
+ plainAccessConfig2.setTopicPerms(Collections.singletonList(DEFAULT_TOPIC + "=" + AclConstants.SUB));
+ plainAccessConfig2.setGroupPerms(Collections.singletonList(DEFAULT_GROUP + "=" + AclConstants.SUB));
+ return plainAccessConfig2;
+ }
+
+ private PlainAccessConfig generateProducerAccessConfig() {
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey(DEFAULT_PRODUCER_AK);
+ plainAccessConfig.setSecretKey(DEFAULT_PRODUCER_SK);
+ plainAccessConfig.setAdmin(false);
+ plainAccessConfig.setDefaultTopicPerm(AclConstants.DENY);
+ plainAccessConfig.setDefaultGroupPerm(AclConstants.DENY);
+ plainAccessConfig.setTopicPerms(Collections.singletonList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
+ return plainAccessConfig;
+ }
+
+ public void validatePullMessage(String topic,
+ String group,
+ AclClientRPCHook aclClientRPCHook,
+ String remoteAddr,
+ PlainAccessValidator plainAccessValidator) {
+ PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
+ pullMessageRequestHeader.setTopic(topic);
+ pullMessageRequestHeader.setConsumerGroup(group);
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE,
+ pullMessageRequestHeader);
+ aclClientRPCHook.doBeforeRequest(remoteAddr, remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(
+ RemotingCommand.decode(buf), remoteAddr);
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+ Assert.fail("Should not throw RemotingCommandException");
+ }
+ }
+
+ public void validateSendMessage(int requestCode,
+ String topic,
+ AclClientRPCHook aclClientRPCHook,
+ String remoteAddr,
+ PlainAccessValidator plainAccessValidator) {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic(topic);
+ RemotingCommand remotingCommand;
+ if (RequestCode.SEND_MESSAGE == requestCode) {
+ remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ } else {
+ remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2,
+ SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
+ }
+
+ aclClientRPCHook.doBeforeRequest(remoteAddr, remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(
+ RemotingCommand.decode(buf), remoteAddr);
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+ Assert.fail("Should not throw RemotingCommandException");
+ }
+ }
+
+ private void checkPlainAccessConfig(final PlainAccessConfig plainAccessConfig,
+ final List plainAccessConfigs) {
+ for (PlainAccessConfig config : plainAccessConfigs) {
+ if (config.getAccessKey().equals(plainAccessConfig.getAccessKey())) {
+ Assert.assertEquals(plainAccessConfig.getSecretKey(), config.getSecretKey());
+ Assert.assertEquals(plainAccessConfig.isAdmin(), config.isAdmin());
+ Assert.assertEquals(plainAccessConfig.getDefaultGroupPerm(), config.getDefaultGroupPerm());
+ Assert.assertEquals(plainAccessConfig.getDefaultGroupPerm(), config.getDefaultGroupPerm());
+ Assert.assertEquals(plainAccessConfig.getWhiteRemoteAddress(), config.getWhiteRemoteAddress());
+ if (null != plainAccessConfig.getTopicPerms()) {
+ Assert.assertNotNull(config.getTopicPerms());
+ Assert.assertTrue(config.getTopicPerms().containsAll(plainAccessConfig.getTopicPerms()));
+ }
+ if (null != plainAccessConfig.getGroupPerms()) {
+ Assert.assertNotNull(config.getGroupPerms());
+ Assert.assertTrue(config.getGroupPerms().containsAll(plainAccessConfig.getGroupPerms()));
+ }
+ }
+ }
+ }
+
+ private void checkDefaultAclFileExists() {
+ boolean isExists = Files.exists(Paths.get(System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf" + File.separator + "plain_acl.yml"));
+ Assert.assertTrue("default acl config file should exist", isExists);
+ }
+
+}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
new file mode 100644
index 00000000000..ef0cffbdcc8
--- /dev/null
+++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
@@ -0,0 +1,1112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.plain;
+
+import com.google.common.base.Joiner;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.AclConstants;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.common.AclConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
+import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PlainAccessValidatorTest {
+
+ private PlainAccessValidator plainAccessValidator;
+ private AclClientRPCHook aclClient;
+ private SessionCredentials sessionCredentials;
+
+ private File confHome;
+
+ @Before
+ public void init() throws IOException {
+ String folder = "conf";
+ confHome = AclTestHelper.copyResources(folder, true);
+ System.setProperty("rocketmq.home.dir", confHome.getAbsolutePath());
+ plainAccessValidator = new PlainAccessValidator();
+ sessionCredentials = new SessionCredentials();
+ sessionCredentials.setAccessKey("RocketMQ");
+ sessionCredentials.setSecretKey("12345678");
+ sessionCredentials.setSecurityToken("87654321");
+ aclClient = new AclClientRPCHook(sessionCredentials);
+ }
+
+ @After
+ public void cleanUp() {
+ AclTestHelper.recursiveDelete(confHome);
+ }
+
+ @Test
+ public void contentTest() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicA");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "127.0.0.1");
+ String signature = AclUtils.calSignature(accessResource.getContent(), sessionCredentials.getSecretKey());
+
+ Assert.assertEquals(accessResource.getSignature(), signature);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+
+ }
+
+ @Test
+ public void validateTest() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+
+ }
+
+ @Test
+ public void validateSendMessageTest() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateSendMessageToRetryTopicTest() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic(MixAll.getRetryTopic("groupB"));
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateSendMessageV2Test() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateSendMessageV2ToRetryTopicTest() {
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic(MixAll.getRetryTopic("groupC"));
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
+ aclClient.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6:9876");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateForAdminCommandWithOutAclRPCHook() {
+ RemotingCommand consumerOffsetAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null);
+ plainAccessValidator.parse(consumerOffsetAdminRequest, "192.168.0.1:9876");
+
+ RemotingCommand subscriptionGroupAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null);
+ plainAccessValidator.parse(subscriptionGroupAdminRequest, "192.168.0.1:9876");
+
+ RemotingCommand delayOffsetAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null);
+ plainAccessValidator.parse(delayOffsetAdminRequest, "192.168.0.1:9876");
+
+ RemotingCommand allTopicConfigAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
+ plainAccessValidator.parse(allTopicConfigAdminRequest, "192.168.0.1:9876");
+
+ }
+
+ @Test
+ public void validatePullMessageTest() {
+ PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
+ pullMessageRequestHeader.setTopic("topicC");
+ pullMessageRequestHeader.setConsumerGroup("groupC");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateConsumeMessageBackTest() {
+ ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader();
+ consumerSendMsgBackRequestHeader.setOriginTopic("topicC");
+ consumerSendMsgBackRequestHeader.setGroup("groupC");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, consumerSendMsgBackRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateQueryMessageTest() {
+ QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();
+ queryMessageRequestHeader.setTopic("topicC");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateQueryMessageByKeyTest() {
+ QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();
+ queryMessageRequestHeader.setTopic("topicC");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ remotingCommand.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, "false");
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1:9876");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateHeartBeatTest() {
+ HeartbeatData heartbeatData = new HeartbeatData();
+ Set producerDataSet = new HashSet<>();
+ Set consumerDataSet = new HashSet<>();
+ Set subscriptionDataSet = new HashSet<>();
+ ProducerData producerData = new ProducerData();
+ producerData.setGroupName("groupB");
+ ConsumerData consumerData = new ConsumerData();
+ consumerData.setGroupName("groupC");
+ SubscriptionData subscriptionData = new SubscriptionData();
+ subscriptionData.setTopic("topicC");
+ producerDataSet.add(producerData);
+ consumerDataSet.add(consumerData);
+ subscriptionDataSet.add(subscriptionData);
+ consumerData.setSubscriptionDataSet(subscriptionDataSet);
+ heartbeatData.setProducerDataSet(producerDataSet);
+ heartbeatData.setConsumerDataSet(consumerDataSet);
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null);
+ remotingCommand.setBody(heartbeatData.encode());
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encode();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateUnRegisterClientTest() {
+ UnregisterClientRequestHeader unregisterClientRequestHeader = new UnregisterClientRequestHeader();
+ unregisterClientRequestHeader.setConsumerGroup("groupB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, unregisterClientRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateGetConsumerListByGroupTest() {
+ GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = new GetConsumerListByGroupRequestHeader();
+ getConsumerListByGroupRequestHeader.setConsumerGroup("groupB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, getConsumerListByGroupRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateUpdateConsumerOffSetTest() {
+ UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = new UpdateConsumerOffsetRequestHeader();
+ updateConsumerOffsetRequestHeader.setConsumerGroup("groupB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, updateConsumerOffsetRequestHeader);
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test(expected = AclException.class)
+ public void validateNullAccessKeyTest() {
+ SessionCredentials sessionCredentials = new SessionCredentials();
+ sessionCredentials.setAccessKey("RocketMQ1");
+ sessionCredentials.setSecretKey("1234");
+ AclClientRPCHook aclClientRPCHook = new AclClientRPCHook(sessionCredentials);
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClientRPCHook.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test(expected = AclException.class)
+ public void validateErrorSecretKeyTest() {
+ SessionCredentials sessionCredentials = new SessionCredentials();
+ sessionCredentials.setAccessKey("RocketMQ");
+ sessionCredentials.setSecretKey("1234");
+ AclClientRPCHook aclClientRPCHook = new AclClientRPCHook(sessionCredentials);
+ SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
+ messageRequestHeader.setTopic("topicB");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
+ aclClientRPCHook.doBeforeRequest("", remotingCommand);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void validateGetAllTopicConfigTest() {
+ String whiteRemoteAddress = "192.168.0.1";
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
+
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), whiteRemoteAddress);
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+
+ Assert.fail("Should not throw IOException");
+ }
+ }
+
+ @Test
+ public void addAccessAclYamlConfigTest() throws InterruptedException {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("rocketmq3");
+ plainAccessConfig.setSecretKey("1234567890");
+ plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
+ plainAccessConfig.setDefaultGroupPerm("PUB");
+ plainAccessConfig.setDefaultTopicPerm("SUB");
+ List topicPerms = new ArrayList<>();
+ topicPerms.add("topicC=PUB|SUB");
+ topicPerms.add("topicB=PUB");
+ plainAccessConfig.setTopicPerms(topicPerms);
+ List groupPerms = new ArrayList<>();
+ groupPerms.add("groupB=PUB|SUB");
+ groupPerms.add("groupC=DENY");
+ plainAccessConfig.setGroupPerms(groupPerms);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+ Thread.sleep(10000);
+
+ Map verifyMap = new HashMap<>();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
+ if (plainAccessConfig1.getAccessKey().equals(plainAccessConfig.getAccessKey())) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig1.getDefaultTopicPerm());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig1.getDefaultGroupPerm());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig1.getTopicPerms());
+ verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig1.getGroupPerms());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_TOPIC_PERM), "SUB");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_GROUP_PERM), "PUB");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.0.*");
+ Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_TOPIC_PERMS)).size(), 2);
+ Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_GROUP_PERMS)).size(), 2);
+
+ String aclFileName = System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml";
+ PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
+ List dataVersions = readableMap.getDataVersion();
+ Assert.assertEquals(1L, dataVersions.get(0).getCounter());
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void getAccessAclYamlConfigTest() {
+ String accessKey = "rocketmq2";
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ Map verifyMap = new HashMap<>();
+ for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
+ if (plainAccessConfig.getAccessKey().equals(accessKey)) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "12345678");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), true);
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.1.*");
+
+ String aclFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ Map dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
+ DataVersion dataVersion = dataVersionMap.get(aclFileName);
+ Assert.assertEquals(0, dataVersion.getCounter().get());
+ }
+
+ @Test
+ public void updateAccessAclYamlConfigTest() throws InterruptedException {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("rocketmq3");
+ plainAccessConfig.setSecretKey("1234567890");
+ plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
+ plainAccessConfig.setDefaultGroupPerm("PUB");
+ plainAccessConfig.setDefaultTopicPerm("SUB");
+ List topicPerms = new ArrayList<>();
+ topicPerms.add("topicC=PUB|SUB");
+ topicPerms.add("topicB=PUB");
+ plainAccessConfig.setTopicPerms(topicPerms);
+ List groupPerms = new ArrayList<>();
+ groupPerms.add("groupB=PUB|SUB");
+ groupPerms.add("groupC=DENY");
+ plainAccessConfig.setGroupPerms(groupPerms);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ Thread.sleep(10000);
+
+ PlainAccessConfig plainAccessConfig1 = new PlainAccessConfig();
+ plainAccessConfig1.setAccessKey("rocketmq3");
+ plainAccessConfig1.setSecretKey("1234567891");
+ plainAccessConfig1.setWhiteRemoteAddress("192.168.0.*");
+ plainAccessConfig1.setDefaultGroupPerm("PUB");
+ plainAccessConfig1.setDefaultTopicPerm("SUB");
+ List topicPerms1 = new ArrayList<>();
+ topicPerms1.add("topicC=PUB|SUB");
+ topicPerms1.add("topicB=PUB");
+ plainAccessConfig1.setTopicPerms(topicPerms1);
+ List groupPerms1 = new ArrayList<>();
+ groupPerms1.add("groupB=PUB|SUB");
+ groupPerms1.add("groupC=DENY");
+ plainAccessConfig1.setGroupPerms(groupPerms1);
+
+ plainAccessValidator.updateAccessConfig(plainAccessConfig1);
+
+ Thread.sleep(10000);
+
+ Map verifyMap = new HashMap<>();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ for (PlainAccessConfig plainAccessConfig2 : plainAccessConfigs) {
+ if (plainAccessConfig2.getAccessKey().equals(plainAccessConfig1.getAccessKey())) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig2.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig2.getDefaultTopicPerm());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig2.getDefaultGroupPerm());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig2.isAdmin());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig2.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig2.getTopicPerms());
+ verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig2.getGroupPerms());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567891");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_TOPIC_PERM), "SUB");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_GROUP_PERM), "PUB");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.0.*");
+ Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_TOPIC_PERMS)).size(), 2);
+ Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_GROUP_PERMS)).size(), 2);
+
+ String aclFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
+ List dataVersions = readableMap.getDataVersion();
+ Assert.assertEquals(2L, dataVersions.get(0).getCounter());
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void deleteAccessAclYamlConfigTest() throws InterruptedException {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("rocketmq3");
+ plainAccessConfig.setSecretKey("1234567890");
+ plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
+ plainAccessConfig.setDefaultGroupPerm("PUB");
+ plainAccessConfig.setDefaultTopicPerm("SUB");
+ List topicPerms = new ArrayList<>();
+ topicPerms.add("topicC=PUB|SUB");
+ topicPerms.add("topicB=PUB");
+ plainAccessConfig.setTopicPerms(topicPerms);
+ List groupPerms = new ArrayList<>();
+ groupPerms.add("groupB=PUB|SUB");
+ groupPerms.add("groupC=DENY");
+ plainAccessConfig.setGroupPerms(groupPerms);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ String accessKey = "rocketmq3";
+ plainAccessValidator.deleteAccessConfig(accessKey);
+ Thread.sleep(10000);
+
+ Map verifyMap = new HashMap<>();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
+ if (plainAccessConfig1.getAccessKey().equals(accessKey)) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig1.getDefaultTopicPerm());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig1.getDefaultGroupPerm());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig1.getTopicPerms());
+ verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig1.getGroupPerms());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.size(), 0);
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void updateGlobalWhiteRemoteAddressesTest() throws InterruptedException {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ List globalWhiteAddrsList = new ArrayList<>();
+ globalWhiteAddrsList.add("192.168.1.*");
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ Assert.assertEquals(plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, null), true);
+
+ String aclFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
+ List dataVersions = readableMap.getDataVersion();
+ Assert.assertEquals(1L, dataVersions.get(0).getCounter());
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void addYamlConfigTest() throws IOException, InterruptedException {
+ String fileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
+ File transport = new File(fileName);
+ transport.delete();
+ transport.createNewFile();
+ FileWriter writer = new FileWriter(transport);
+ writer.write("accounts:\r\n");
+ writer.write("- accessKey: watchrocketmqx\r\n");
+ writer.write(" secretKey: 12345678\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: true\r\n");
+ writer.flush();
+ writer.close();
+
+ Thread.sleep(1000);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ Map verifyMap = new HashMap<>();
+ for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
+ if (plainAccessConfig.getAccessKey().equals("watchrocketmqx")) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "12345678");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), true);
+
+ Map dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
+ DataVersion dataVersion = dataVersionMap.get(fileName);
+ Assert.assertEquals(0, dataVersion.getCounter().get());
+
+ transport.delete();
+ }
+
+ @Test
+ public void updateAccessAnotherAclYamlConfigTest() throws IOException, InterruptedException {
+ String fileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
+ File transport = new File(fileName);
+ transport.delete();
+ transport.createNewFile();
+ FileWriter writer = new FileWriter(transport);
+ writer.write("accounts:\r\n");
+ writer.write("- accessKey: watchrocketmqy\r\n");
+ writer.write(" secretKey: 12345678\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: true\r\n");
+ writer.write("- accessKey: watchrocketmqx\r\n");
+ writer.write(" secretKey: 123456781\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: true\r\n");
+ writer.flush();
+ writer.close();
+
+ Thread.sleep(1000);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("watchrocketmqy");
+ plainAccessConfig.setSecretKey("1234567890");
+ plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
+ plainAccessConfig.setAdmin(false);
+
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ Thread.sleep(1000);
+
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ Map verifyMap = new HashMap<>();
+ for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
+ if (plainAccessConfig1.getAccessKey().equals("watchrocketmqy")) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
+
+ Map dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
+ DataVersion dataVersion = dataVersionMap.get(fileName);
+ Assert.assertEquals(1, dataVersion.getCounter().get());
+
+ transport.delete();
+
+ }
+
+ @Test(expected = AclException.class)
+ public void createAndUpdateAccessAclNullSkExceptionTest() {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("RocketMQ33");
+ // secret key is null
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void addAccessDefaultAclYamlConfigTest() throws InterruptedException {
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("watchrocketmqh");
+ plainAccessConfig.setSecretKey("1234567890");
+ plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
+ plainAccessConfig.setAdmin(false);
+
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ Thread.sleep(10000);
+
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ Map verifyMap = new HashMap<>();
+ for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
+ if (plainAccessConfig1.getAccessKey().equals("watchrocketmqh")) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
+ Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
+
+ PlainAccessData readableMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
+ List dataVersions = readableMap.getDataVersion();
+ Assert.assertEquals(1L, dataVersions.get(0).getCounter());
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void deleteAccessAnotherAclYamlConfigTest() throws IOException, InterruptedException {
+ String fileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
+ File transport = new File(fileName);
+ transport.delete();
+ transport.createNewFile();
+ FileWriter writer = new FileWriter(transport);
+ writer.write("accounts:\r\n");
+ writer.write("- accessKey: watchrocketmqx\r\n");
+ writer.write(" secretKey: 12345678\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: true\r\n");
+ writer.write("- accessKey: watchrocketmqy\r\n");
+ writer.write(" secretKey: 1234567890\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: false\r\n");
+ writer.flush();
+ writer.close();
+
+ Thread.sleep(1000);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.deleteAccessConfig("watchrocketmqx");
+ Thread.sleep(10000);
+
+ Map verifyMap = new HashMap<>();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ List plainAccessConfigs = aclConfig.getPlainAccessConfigs();
+ for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
+ if (plainAccessConfig.getAccessKey().equals("watchrocketmqx")) {
+ verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig.getDefaultTopicPerm());
+ verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig.getDefaultGroupPerm());
+ verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
+ verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
+ verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig.getTopicPerms());
+ verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig.getGroupPerms());
+ }
+ }
+
+ Assert.assertEquals(verifyMap.size(), 0);
+
+ transport.delete();
+ }
+
+ @Test
+ public void getAllAclConfigTest() {
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
+ Assert.assertEquals(aclConfig.getGlobalWhiteAddrs().size(), 4);
+ Assert.assertEquals(aclConfig.getPlainAccessConfigs().size(), 2);
+ }
+
+ @Test
+ public void updateAccessConfigEmptyPermListTest() {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ String accessKey = "updateAccessConfigEmptyPerm";
+ plainAccessConfig.setAccessKey(accessKey);
+ plainAccessConfig.setSecretKey("123456789111");
+ plainAccessConfig.setTopicPerms(Collections.singletonList("topicB=PUB"));
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ plainAccessConfig.setTopicPerms(new ArrayList<>());
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ List plainAccessConfigs = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs();
+ for (int i = 0; i < plainAccessConfigs.size(); i++) {
+ PlainAccessConfig plainAccessConfig1 = plainAccessConfigs.get(i);
+ if (plainAccessConfig1.getAccessKey() == accessKey) {
+ Assert.assertEquals(0, plainAccessConfig1.getTopicPerms().size());
+ }
+ }
+
+ plainAccessValidator.deleteAccessConfig(accessKey);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void updateAccessConfigEmptyWhiteRemoteAddressTest() {
+ String backupFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ String accessKey = "updateAccessConfigEmptyWhiteRemoteAddress";
+ plainAccessConfig.setAccessKey(accessKey);
+ plainAccessConfig.setSecretKey("123456789111");
+ plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ plainAccessConfig.setWhiteRemoteAddress("");
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+ List plainAccessConfigs = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs();
+ for (int i = 0; i < plainAccessConfigs.size(); i++) {
+ PlainAccessConfig plainAccessConfig1 = plainAccessConfigs.get(i);
+ if (plainAccessConfig1.getAccessKey() == accessKey) {
+ Assert.assertEquals("", plainAccessConfig1.getWhiteRemoteAddress());
+ }
+ }
+
+ plainAccessValidator.deleteAccessConfig(accessKey);
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ }
+
+ @Test
+ public void deleteAccessAclToEmptyTest() {
+ final String bakAclFileProp = System.getProperty("rocketmq.acl.plain.file");
+ System.setProperty("rocketmq.acl.plain.file", "conf/empty.yml".replace("/", File.separator));
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("deleteAccessAclToEmpty");
+ plainAccessConfig.setSecretKey("12345678");
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+ boolean success = plainAccessValidator.deleteAccessConfig("deleteAccessAclToEmpty");
+ if (null != bakAclFileProp) {
+ System.setProperty("rocketmq.acl.plain.file", bakAclFileProp);
+ } else {
+ System.clearProperty("rocketmq.acl.plain.file");
+ }
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testValidateAfterUpdateAccessConfig() throws NoSuchFieldException, IllegalAccessException {
+ String targetFileName = System.getProperty("rocketmq.home.dir")
+ + File.separator + "conf/update.yml".replace("/", File.separator);
+ System.setProperty("rocketmq.acl.plain.file", "conf/update.yml".replace("/", File.separator));
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ String accessKey = "updateAccessConfig";
+ String secretKey = "123456789111";
+ plainAccessConfig.setAccessKey(accessKey);
+ plainAccessConfig.setSecretKey(secretKey);
+ plainAccessConfig.setAdmin(true);
+ // update
+ plainAccessValidator.updateAccessConfig(plainAccessConfig);
+ // call load
+ Class clazz = PlainAccessValidator.class;
+ Field f = clazz.getDeclaredField("aclPlugEngine");
+ f.setAccessible(true);
+ PlainPermissionManager aclPlugEngine = (PlainPermissionManager) f.get(plainAccessValidator);
+ aclPlugEngine.load(targetFileName);
+
+ // call validate
+ PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
+ pullMessageRequestHeader.setTopic("topicC");
+ pullMessageRequestHeader.setConsumerGroup("consumerGroupA");
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
+
+ AclClientRPCHook aclClient = new AclClientRPCHook(new SessionCredentials(accessKey, secretKey));
+ aclClient.doBeforeRequest("", remotingCommand);
+ ByteBuffer buf = remotingCommand.encodeHeader();
+ buf.getInt();
+ buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
+ buf.position(0);
+ try {
+ PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "1.1.1.1:9876");
+ plainAccessValidator.validate(accessResource);
+ } catch (RemotingCommandException e) {
+ e.printStackTrace();
+ Assert.fail("Should not throw IOException");
+ } finally {
+ System.setProperty("rocketmq.acl.plain.file", "conf/plain_acl.yml".replace("/", File.separator));
+ }
+ }
+
+ /**
+ * Fixme: this test case is not thread safe. The design itself is buggy.
+ * @throws IOException
+ */
+ @Test
+ public void testUpdateSpecifiedAclFileGlobalWhiteAddrsConfig() throws IOException {
+ String folder = "update_global_white_addr";
+ File home = AclTestHelper.copyResources(folder);
+ System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
+ System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml".replace("/", File.separator));
+
+ String targetFileName = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "plain_acl.yml"});
+ PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
+
+ String targetFileName1 = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "acl", "plain_acl.yml"});
+ PlainAccessData backUpAclConfigMap1 = AclUtils.getYamlDataObject(targetFileName1, PlainAccessData.class);
+
+ String targetFileName2 = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "acl", "empty.yml"});
+ PlainAccessData backUpAclConfigMap2 = AclUtils.getYamlDataObject(targetFileName2, PlainAccessData.class);
+
+ PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+ List globalWhiteAddrsList1 = new ArrayList<>();
+ globalWhiteAddrsList1.add("10.10.154.1");
+ List globalWhiteAddrsList2 = new ArrayList<>();
+ globalWhiteAddrsList2.add("10.10.154.2");
+ List globalWhiteAddrsList3 = new ArrayList<>();
+ globalWhiteAddrsList3.add("10.10.154.3");
+
+ //Test parameter p is null
+ plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList1, null);
+ String defaultAclFile = targetFileName;
+ PlainAccessData defaultAclFileMap = AclUtils.getYamlDataObject(defaultAclFile, PlainAccessData.class);
+ List defaultAclFileGlobalWhiteAddrList = defaultAclFileMap.getGlobalWhiteRemoteAddresses();
+ Assert.assertTrue(defaultAclFileGlobalWhiteAddrList.contains("10.10.154.1"));
+ //Test parameter p is not null
+ plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList2, targetFileName1);
+ PlainAccessData aclFileMap1 = AclUtils.getYamlDataObject(targetFileName1, PlainAccessData.class);
+ List aclFileGlobalWhiteAddrList1 = aclFileMap1.getGlobalWhiteRemoteAddresses();
+ Assert.assertTrue(aclFileGlobalWhiteAddrList1.contains("10.10.154.2"));
+ //Test parameter p is not null, but the file does not have globalWhiteRemoteAddresses
+ plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList3, targetFileName2);
+ PlainAccessData aclFileMap2 = AclUtils.getYamlDataObject(targetFileName2, PlainAccessData.class);
+ List aclFileGlobalWhiteAddrList2 = aclFileMap2.getGlobalWhiteRemoteAddresses();
+ Assert.assertTrue(aclFileGlobalWhiteAddrList2.contains("10.10.154.3"));
+
+ AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
+ AclUtils.writeDataObject(targetFileName1, backUpAclConfigMap1);
+ AclUtils.writeDataObject(targetFileName2, backUpAclConfigMap2);
+
+ AclTestHelper.recursiveDelete(home);
+ }
+
+
+}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
new file mode 100644
index 00000000000..941d8c77923
--- /dev/null
+++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.plain;
+
+import com.google.common.base.Joiner;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.acl.common.AclConstants;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.Permission;
+import org.apache.rocketmq.common.AclConfig;
+import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.assertj.core.api.Assertions;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class PlainPermissionManagerTest {
+
+ PlainPermissionManager plainPermissionManager;
+ PlainAccessResource pubPlainAccessResource;
+ PlainAccessResource subPlainAccessResource;
+ PlainAccessResource anyPlainAccessResource;
+ PlainAccessResource denyPlainAccessResource;
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+ Set adminCode = new HashSet<>();
+
+ private static final String DEFAULT_TOPIC = "topic-acl";
+
+ private File confHome;
+
+ @Before
+ public void init() throws NoSuchFieldException, SecurityException, IOException {
+ // UPDATE_AND_CREATE_TOPIC
+ adminCode.add(17);
+ // UPDATE_BROKER_CONFIG
+ adminCode.add(25);
+ // DELETE_TOPIC_IN_BROKER
+ adminCode.add(215);
+ // UPDATE_AND_CREATE_SUBSCRIPTIONGROUP
+ adminCode.add(200);
+ // DELETE_SUBSCRIPTIONGROUP
+ adminCode.add(207);
+
+ pubPlainAccessResource = clonePlainAccessResource(Permission.PUB);
+ subPlainAccessResource = clonePlainAccessResource(Permission.SUB);
+ anyPlainAccessResource = clonePlainAccessResource(Permission.ANY);
+ denyPlainAccessResource = clonePlainAccessResource(Permission.DENY);
+
+ String folder = "conf";
+ confHome = AclTestHelper.copyResources(folder, true);
+ System.setProperty("rocketmq.home.dir", confHome.getAbsolutePath());
+ plainPermissionManager = new PlainPermissionManager();
+ }
+
+ public PlainAccessResource clonePlainAccessResource(byte perm) {
+ PlainAccessResource painAccessResource = new PlainAccessResource();
+ painAccessResource.setAccessKey("RocketMQ");
+ painAccessResource.setSecretKey("12345678");
+ painAccessResource.setWhiteRemoteAddress("127.0." + perm + ".*");
+ painAccessResource.setDefaultGroupPerm(perm);
+ painAccessResource.setDefaultTopicPerm(perm);
+ painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupA"), Permission.PUB);
+ painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupB"), Permission.SUB);
+ painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupC"), Permission.ANY);
+ painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupD"), Permission.DENY);
+
+ painAccessResource.addResourceAndPerm("topicA", Permission.PUB);
+ painAccessResource.addResourceAndPerm("topicB", Permission.SUB);
+ painAccessResource.addResourceAndPerm("topicC", Permission.ANY);
+ painAccessResource.addResourceAndPerm("topicD", Permission.DENY);
+ return painAccessResource;
+ }
+
+ @Test
+ public void buildPlainAccessResourceTest() {
+ PlainAccessResource plainAccessResource = null;
+ PlainAccessConfig plainAccess = new PlainAccessConfig();
+
+ plainAccess.setAccessKey("RocketMQ");
+ plainAccess.setSecretKey("12345678");
+ plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
+ Assert.assertEquals(plainAccessResource.getAccessKey(), "RocketMQ");
+ Assert.assertEquals(plainAccessResource.getSecretKey(), "12345678");
+
+ plainAccess.setWhiteRemoteAddress("127.0.0.1");
+ plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
+ Assert.assertEquals(plainAccessResource.getWhiteRemoteAddress(), "127.0.0.1");
+
+ plainAccess.setAdmin(true);
+ plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
+ Assert.assertEquals(plainAccessResource.isAdmin(), true);
+
+ List groups = new ArrayList<>();
+ groups.add("groupA=DENY");
+ groups.add("groupB=PUB|SUB");
+ groups.add("groupC=PUB");
+ plainAccess.setGroupPerms(groups);
+ plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
+ Map resourcePermMap = plainAccessResource.getResourcePermMap();
+ Assert.assertEquals(resourcePermMap.size(), 3);
+
+ Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA")).byteValue(), Permission.DENY);
+ Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")).byteValue(), Permission.PUB | Permission.SUB);
+ Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC")).byteValue(), Permission.PUB);
+
+ List topics = new ArrayList<>();
+ topics.add("topicA=DENY");
+ topics.add("topicB=PUB|SUB");
+ topics.add("topicC=PUB");
+ plainAccess.setTopicPerms(topics);
+ plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
+ resourcePermMap = plainAccessResource.getResourcePermMap();
+ Assert.assertEquals(resourcePermMap.size(), 6);
+
+ Assert.assertEquals(resourcePermMap.get("topicA").byteValue(), Permission.DENY);
+ Assert.assertEquals(resourcePermMap.get("topicB").byteValue(), Permission.PUB | Permission.SUB);
+ Assert.assertEquals(resourcePermMap.get("topicC").byteValue(), Permission.PUB);
+ }
+
+ @Test(expected = AclException.class)
+ public void checkPermAdmin() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setRequestCode(17);
+ plainPermissionManager.checkPerm(plainAccessResource, pubPlainAccessResource);
+ }
+
+ @Test
+ public void checkPerm() {
+
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.addResourceAndPerm("topicA", Permission.PUB);
+ plainPermissionManager.checkPerm(plainAccessResource, pubPlainAccessResource);
+ plainAccessResource.addResourceAndPerm("topicB", Permission.SUB);
+ plainPermissionManager.checkPerm(plainAccessResource, anyPlainAccessResource);
+
+ plainAccessResource = new PlainAccessResource();
+ plainAccessResource.addResourceAndPerm("topicB", Permission.SUB);
+ plainPermissionManager.checkPerm(plainAccessResource, subPlainAccessResource);
+ plainAccessResource.addResourceAndPerm("topicA", Permission.PUB);
+ plainPermissionManager.checkPerm(plainAccessResource, anyPlainAccessResource);
+
+ }
+
+ @Test(expected = AclException.class)
+ public void checkErrorPermDefaultValueNotMatch() {
+
+ plainAccessResource = new PlainAccessResource();
+ plainAccessResource.addResourceAndPerm("topicF", Permission.PUB);
+ plainPermissionManager.checkPerm(plainAccessResource, subPlainAccessResource);
+ }
+
+ @Test(expected = AclException.class)
+ public void accountNullTest() {
+ plainAccessConfig.setAccessKey(null);
+ plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
+ }
+
+ @Test(expected = AclException.class)
+ public void accountThanTest() {
+ plainAccessConfig.setAccessKey("123");
+ plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
+ }
+
+ @Test(expected = AclException.class)
+ public void passWordtNullTest() {
+ plainAccessConfig.setAccessKey(null);
+ plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
+ }
+
+ @Test(expected = AclException.class)
+ public void passWordThanTest() {
+ plainAccessConfig.setSecretKey("123");
+ plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void cleanAuthenticationInfoTest() throws IllegalAccessException {
+ // PlainPermissionManager.addPlainAccessResource(plainAccessResource);
+ Map> plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
+ Assert.assertFalse(plainAccessResourceMap.isEmpty());
+
+ plainPermissionManager.clearPermissionInfo();
+ plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
+ Assert.assertTrue(plainAccessResourceMap.isEmpty());
+ }
+
+ @Test
+ public void isWatchStartTest() {
+
+ PlainPermissionManager plainPermissionManager = new PlainPermissionManager();
+ Assert.assertTrue(plainPermissionManager.isWatchStart());
+ }
+
+ @Test
+ public void testWatch() throws IOException, IllegalAccessException, InterruptedException {
+ String fileName = Joiner.on(File.separator).join(new String[]{System.getProperty("rocketmq.home.dir"), "conf", "acl", "plain_acl_test.yml"});
+ File transport = new File(fileName);
+ transport.delete();
+ transport.createNewFile();
+ FileWriter writer = new FileWriter(transport);
+ writer.write("accounts:\r\n");
+ writer.write("- accessKey: watchrocketmqx\r\n");
+ writer.write(" secretKey: 12345678\r\n");
+ writer.write(" whiteRemoteAddress: 127.0.0.1\r\n");
+ writer.write(" admin: true\r\n");
+ writer.flush();
+ writer.close();
+
+ Thread.sleep(1000);
+
+ PlainPermissionManager plainPermissionManager = new PlainPermissionManager();
+ Assert.assertTrue(plainPermissionManager.isWatchStart());
+
+ Map accessKeyTable = (Map) FieldUtils.readDeclaredField(plainPermissionManager, "accessKeyTable", true);
+ String aclFileName = accessKeyTable.get("watchrocketmqx");
+ {
+ Map> plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
+ PlainAccessResource accessResource = plainAccessResourceMap.get(aclFileName).get("watchrocketmqx");
+ Assert.assertNotNull(accessResource);
+ Assert.assertEquals(accessResource.getSecretKey(), "12345678");
+ Assert.assertTrue(accessResource.isAdmin());
+
+ }
+
+ PlainAccessData updatedMap = AclUtils.getYamlDataObject(fileName, PlainAccessData.class);
+ List accounts = updatedMap.getAccounts();
+ accounts.get(0).setAccessKey("watchrocketmq1y");
+ accounts.get(0).setSecretKey("88888888");
+ accounts.get(0).setAdmin(false);
+ // Update file and flush to yaml file
+ AclUtils.writeDataObject(fileName, updatedMap);
+
+ Thread.sleep(10000);
+ {
+ Map> plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
+ PlainAccessResource accessResource = plainAccessResourceMap.get(aclFileName).get("watchrocketmq1y");
+ Assert.assertNotNull(accessResource);
+ Assert.assertEquals(accessResource.getSecretKey(), "88888888");
+ Assert.assertFalse(accessResource.isAdmin());
+
+ }
+ transport.delete();
+ }
+
+ @Test
+ public void updateAccessConfigTest() {
+ Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(null));
+
+ plainAccessConfig.setAccessKey("admin_test");
+ // Invalid parameter
+ plainAccessConfig.setSecretKey("123456");
+ plainAccessConfig.setAdmin(true);
+ Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(plainAccessConfig));
+
+ plainAccessConfig.setSecretKey("12345678");
+ // Invalid parameter
+ plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA!SUB"));
+ Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(plainAccessConfig));
+
+ // first update
+ plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
+ plainPermissionManager.updateAccessConfig(plainAccessConfig);
+
+ // second update
+ plainAccessConfig.setTopicPerms(Lists.newArrayList("topicA=SUB"));
+ plainPermissionManager.updateAccessConfig(plainAccessConfig);
+ }
+
+ @Test
+ public void getAllAclFilesTest() {
+ final List notExistList = plainPermissionManager.getAllAclFiles("aa/bb");
+ Assertions.assertThat(notExistList).isEmpty();
+ final List files = plainPermissionManager.getAllAclFiles(confHome.getAbsolutePath());
+ Assertions.assertThat(files).isNotEmpty();
+ }
+
+ @Test
+ public void loadTest() {
+ plainPermissionManager.load();
+ final Map map = plainPermissionManager.getDataVersionMap();
+ Assertions.assertThat(map).isNotEmpty();
+ }
+
+ @Test
+ public void updateAclConfigFileVersionTest() {
+ String aclFileName = "test_plain_acl";
+ PlainAccessData updateAclConfigMap = new PlainAccessData();
+ List versionElement = new ArrayList<>();
+ PlainAccessData.DataVersion accountsMap = new PlainAccessData.DataVersion();
+ accountsMap.setCounter(1);
+ accountsMap.setTimestamp(System.currentTimeMillis());
+ versionElement.add(accountsMap);
+
+ updateAclConfigMap.setDataVersion(versionElement);
+ final PlainAccessData map = plainPermissionManager.updateAclConfigFileVersion(aclFileName, updateAclConfigMap);
+ final List version = map.getDataVersion();
+ Assert.assertEquals(2L, version.get(0).getCounter());
+ }
+
+ @Test
+ public void createAclAccessConfigMapTest() {
+ PlainAccessConfig existedAccountMap = new PlainAccessConfig();
+ plainAccessConfig.setAccessKey("admin123");
+ plainAccessConfig.setSecretKey("12345678");
+ plainAccessConfig.setWhiteRemoteAddress("192.168.1.1");
+ plainAccessConfig.setAdmin(false);
+ plainAccessConfig.setDefaultGroupPerm(AclConstants.SUB_PUB);
+ plainAccessConfig.setTopicPerms(Arrays.asList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
+ plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
+
+ final PlainAccessConfig map = plainPermissionManager.createAclAccessConfigMap(existedAccountMap, plainAccessConfig);
+ Assert.assertEquals(AclConstants.SUB_PUB, map.getDefaultGroupPerm());
+ Assert.assertEquals("groupA=SUB", map.getGroupPerms().get(0));
+ Assert.assertEquals("12345678", map.getSecretKey());
+ Assert.assertEquals("admin123", map.getAccessKey());
+ Assert.assertEquals("192.168.1.1", map.getWhiteRemoteAddress());
+ Assert.assertEquals("topic-acl=PUB", map.getTopicPerms().get(0));
+ Assert.assertEquals(false, map.isAdmin());
+ }
+
+ @Test
+ public void deleteAccessConfigTest() throws InterruptedException {
+ // delete not exist accessConfig
+ final boolean flag1 = plainPermissionManager.deleteAccessConfig("test_delete");
+ assert !flag1;
+
+ plainAccessConfig.setAccessKey("test_delete");
+ plainAccessConfig.setSecretKey("12345678");
+ plainAccessConfig.setWhiteRemoteAddress("192.168.1.1");
+ plainAccessConfig.setAdmin(false);
+ plainAccessConfig.setDefaultGroupPerm(AclConstants.SUB_PUB);
+ plainAccessConfig.setTopicPerms(Arrays.asList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
+ plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
+ plainPermissionManager.updateAccessConfig(plainAccessConfig);
+
+ //delete existed accessConfig
+ final boolean flag2 = plainPermissionManager.deleteAccessConfig("test_delete");
+ assert flag2;
+
+ }
+
+ @Test
+ public void updateGlobalWhiteAddrsConfigTest() {
+ final boolean flag = plainPermissionManager.updateGlobalWhiteAddrsConfig(Lists.newArrayList("192.168.1.2"));
+ assert flag;
+ final AclConfig config = plainPermissionManager.getAllAclConfig();
+ Assert.assertEquals(true, config.getGlobalWhiteAddrs().contains("192.168.1.2"));
+ }
+
+}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java
new file mode 100644
index 00000000000..df7dd0c5460
--- /dev/null
+++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.plain;
+
+import org.apache.rocketmq.acl.common.AclException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class RemoteAddressStrategyTest {
+
+ RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory();
+
+ @Test
+ public void netAddressStrategyFactoryExceptionTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource).getClass(),
+ RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class);
+ }
+
+ @Test
+ public void netAddressStrategyFactoryTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+
+ plainAccessResource.setWhiteRemoteAddress("*");
+ RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
+
+ plainAccessResource.setWhiteRemoteAddress("*.*.*.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.1-20.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class);
+
+// IPv6 test
+ plainAccessResource.setWhiteRemoteAddress("*:*:*:*:*:*:*:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:326b");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261,1050::0005:0600:300c:3262,1050::0005:0600:300c:3263");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:{1,2,3}");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-200");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0005:0600:300c:3261:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-20:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
+ }
+
+ @Test(expected = AclException.class)
+ public void verifyTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("256.0.0.1");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("::1ggg");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ }
+
+ @Test
+ public void nullNetAddressStrategyTest() {
+ boolean isMatch = RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY.match(new PlainAccessResource());
+ Assert.assertTrue(isMatch);
+ }
+
+ @Test
+ public void blankNetAddressStrategyTest() {
+ boolean isMatch = RemoteAddressStrategyFactory.BLANK_NET_ADDRESS_STRATEGY.match(new PlainAccessResource());
+ Assert.assertFalse(isMatch);
+ }
+
+ @Test
+ public void oneNetAddressStrategyTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
+ RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("");
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.2");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+// Ipv6 test
+ plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("::1");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("::2");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("::1");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("0000:0000:0000:0000:0000:0000:0000:0001");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+ }
+
+ @Test
+ public void multipleNetAddressStrategyTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3");
+ RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ multipleNetAddressStrategyTest(remoteAddressStrategy);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ multipleNetAddressStrategyTest(remoteAddressStrategy);
+
+ plainAccessResource.setWhiteRemoteAddress("192.100-150.*.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("192.130.0.2");
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1,1050::0005:0600:300c:2,1050::0005:0600:300c:3");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ multipleIPv6NetAddressStrategyTest(remoteAddressStrategy);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:{1,2,3}");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ multipleIPv6NetAddressStrategyTest(remoteAddressStrategy);
+
+ }
+
+ @Test(expected = AclException.class)
+ public void multipleNetAddressStrategyExceptionTest() {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1,2,3}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("::1,2,3}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("192.168.1.{1}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("192.168.1.{1,2}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("192.168.{1}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("{192.168.1}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ plainAccessResource.setWhiteRemoteAddress("{192.168.1.1}");
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ }
+
+ private void multipleNetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.2");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.3");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.4");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.0");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ }
+
+ private void multipleIPv6NetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:1");
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:2");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:3");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertTrue(match);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:4");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:0");
+ match = remoteAddressStrategy.match(plainAccessResource);
+ Assert.assertFalse(match);
+
+ }
+
+ @Test
+ public void rangeNetAddressStrategyTest() {
+ String head = "127.0.0.";
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200");
+ RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeNetAddressStrategyTest(remoteAddressStrategy, head, 1, 200, true);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.0.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeNetAddressStrategyTest(remoteAddressStrategy, head, 0, 255, true);
+
+ plainAccessResource.setWhiteRemoteAddress("127.0.1-200.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeNetAddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200);
+
+ plainAccessResource.setWhiteRemoteAddress("127.*.*.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeNetAddressStrategyTest(remoteAddressStrategy, head, 0, 255, true);
+
+ plainAccessResource.setWhiteRemoteAddress("127.1-150.*.*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeNetAddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200);
+
+// IPv6 test
+ head = "1050::0005:0600:300c:";
+ plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "1", "200", true);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true);
+
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:3001:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false);
+
+ head = "1050::0005:0600:300c:1:";
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true);
+
+ head = "1050::0005:0600:300c:201:";
+ plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*");
+ remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false);
+
+ }
+
+ private void rangeNetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start,
+ int end,
+ boolean isFalse) {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ for (int i = -10; i < 300; i++) {
+ plainAccessResource.setWhiteRemoteAddress(head + i);
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ if (isFalse && i >= start && i <= end) {
+ Assert.assertTrue(match);
+ continue;
+ }
+ Assert.assertFalse(match);
+
+ }
+ }
+
+ private void rangeNetAddressStrategyThirdlyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start,
+ int end) {
+ String newHead;
+ for (int i = -10; i < 300; i++) {
+ newHead = head + i;
+ if (i >= start && i <= end) {
+ rangeNetAddressStrategyTest(remoteAddressStrategy, newHead, 0, 255, false);
+ }
+ }
+ }
+
+ private void rangeIPv6NetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, String start,
+ String end,
+ boolean isFalse) {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ for (int i = -10; i < 65536 + 100; i++) {
+ String hex = Integer.toHexString(i);
+ plainAccessResource.setWhiteRemoteAddress(head + hex);
+ boolean match = remoteAddressStrategy.match(plainAccessResource);
+ int startNum = Integer.parseInt(start, 16);
+ int endNum = Integer.parseInt(end, 16);
+ if (isFalse && i >= startNum && i <= endNum) {
+ Assert.assertTrue(match);
+ continue;
+ }
+ Assert.assertFalse(match);
+
+ }
+ }
+
+ @Test(expected = AclException.class)
+ public void rangeNetAddressStrategyExceptionStartGreaterEndTest() {
+ rangeNetAddressStrategyExceptionTest("127.0.0.2-1");
+ }
+
+ @Test(expected = AclException.class)
+ public void rangeNetAddressStrategyExceptionScopeTest() {
+ rangeNetAddressStrategyExceptionTest("127.0.0.-1-200");
+ }
+
+ @Test(expected = AclException.class)
+ public void rangeNetAddressStrategyExceptionScopeTwoTest() {
+ rangeNetAddressStrategyExceptionTest("127.0.0.0-256");
+ }
+
+ private void rangeNetAddressStrategyExceptionTest(String netAddress) {
+ PlainAccessResource plainAccessResource = new PlainAccessResource();
+ plainAccessResource.setWhiteRemoteAddress(netAddress);
+ remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
+ }
+
+}
diff --git a/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml b/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml
new file mode 100644
index 00000000000..28a8c488805
--- /dev/null
+++ b/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+accounts:
+ - accessKey: rocketmq3
+ secretKey: 12345678
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: DENY
+ topicPerms:
+ - topicA=PUB
+ - topicB=SUB
+ - topicC=PUB|SUB
+ - topicD=DENY
+ groupPerms:
+ - groupB=SUB
+ - groupC=PUB|SUB
+ - groupD=DENY
+
diff --git a/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml b/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml
new file mode 100644
index 00000000000..cf4ea7f4a5b
--- /dev/null
+++ b/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## no global white addresses in this file, define them in ../plain_acl.yml
+accounts:
+ - accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ # the group should convert to retry topic
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+
+ - accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ # if it is admin, it could access all resources
+ admin: true
+
diff --git a/acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml b/acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml
new file mode 100644
index 00000000000..41afea097ae
--- /dev/null
+++ b/acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+ - 10.10.103.*
+ - 192.168.0.*
+
diff --git a/acl/src/test/resources/conf/acl/plain_acl.yml b/acl/src/test/resources/conf/acl/plain_acl.yml
new file mode 100644
index 00000000000..34e46696d8e
--- /dev/null
+++ b/acl/src/test/resources/conf/acl/plain_acl.yml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+ - 10.10.103.*
+ - 192.168.0.*
+
+accounts:
+ - accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ # the group should convert to retry topic
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+
+ - accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ # if it is admin, it could access all resources
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl.yml b/acl/src/test/resources/conf/plain_acl.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_bak.yml b/acl/src/test/resources/conf/plain_acl_bak.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_bak.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_correct.yml b/acl/src/test/resources/conf/plain_acl_correct.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_correct.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_delete.yml b/acl/src/test/resources/conf/plain_acl_delete.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_delete.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml b/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_incomplete.yml b/acl/src/test/resources/conf/plain_acl_incomplete.yml
new file mode 100644
index 00000000000..0a6bdde7072
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_incomplete.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+- accessKey: rocketmq2
+ secretKey:
+ whiteRemoteAddress: 192.168.1.*
+ # if it is admin, it could access all resources
+ admin: true
\ No newline at end of file
diff --git a/acl/src/test/resources/conf/plain_acl_update_create.yml b/acl/src/test/resources/conf/plain_acl_update_create.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_update_create.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml b/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml
new file mode 100644
index 00000000000..939f7c98ca6
--- /dev/null
+++ b/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
\ No newline at end of file
diff --git a/acl/src/test/resources/conf/watch/plain_acl_watch.yml b/acl/src/test/resources/conf/watch/plain_acl_watch.yml
new file mode 100644
index 00000000000..9d2c3954941
--- /dev/null
+++ b/acl/src/test/resources/conf/watch/plain_acl_watch.yml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+accounts:
+- accessKey: watchrocketmq
+ secretKey: 12345678
+ whiteRemoteAddress: 127.0.0.1
+ admin: true
+- accessKey: watchrocketmq1
+ secretKey: 88888888
+ whiteRemoteAddress: 127.0.0.1
+ admin: false
diff --git a/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml b/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml
new file mode 100644
index 00000000000..6ade46723b7
--- /dev/null
+++ b/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+globalWhiteRemoteAddresses:
+ - 10.10.103.*
+ - 192.168.0.*
diff --git a/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml b/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml
new file mode 100644
index 00000000000..cf4ea7f4a5b
--- /dev/null
+++ b/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## no global white addresses in this file, define them in ../plain_acl.yml
+accounts:
+ - accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ # the group should convert to retry topic
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+
+ - accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ # if it is admin, it could access all resources
+ admin: true
+
diff --git a/acl/src/test/resources/rmq.logback-test.xml b/acl/src/test/resources/rmq.logback-test.xml
new file mode 100644
index 00000000000..8695d52d57c
--- /dev/null
+++ b/acl/src/test/resources/rmq.logback-test.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+ %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml b/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml
new file mode 100644
index 00000000000..52ff50c2bef
--- /dev/null
+++ b/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+accounts: []
diff --git a/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml b/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml
new file mode 100644
index 00000000000..59bd6d4ff29
--- /dev/null
+++ b/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+globalWhiteRemoteAddresses:
+- 10.10.103.*
+- 192.168.0.*
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml b/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml
new file mode 100644
index 00000000000..64c6e34169c
--- /dev/null
+++ b/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## suggested format
+
+accounts:
+- accessKey: RocketMQ
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.0.*
+ admin: false
+ defaultTopicPerm: DENY
+ defaultGroupPerm: SUB
+ topicPerms:
+ - topicA=DENY
+ - topicB=PUB|SUB
+ - topicC=SUB
+ groupPerms:
+ - groupA=DENY
+ - groupB=SUB
+ - groupC=SUB
+- accessKey: rocketmq2
+ secretKey: 12345678
+ whiteRemoteAddress: 192.168.1.*
+ admin: true
diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel
new file mode 100644
index 00000000000..a9fd83fea0b
--- /dev/null
+++ b/bazel/BUILD.bazel
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
diff --git a/bazel/GenTestRules.bzl b/bazel/GenTestRules.bzl
new file mode 100644
index 00000000000..fb9b6991de0
--- /dev/null
+++ b/bazel/GenTestRules.bzl
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Generate java test rules from given test_files.
+
+Instead of having to create one test rule per test in the BUILD file, this rule
+provides a handy way to create a bunch of test rules for the specified test
+files.
+
+"""
+
+def GenTestRules(
+ name,
+ test_files,
+ deps,
+ exclude_tests = [],
+ default_test_size = "small",
+ small_tests = [],
+ medium_tests = [],
+ large_tests = [],
+ enormous_tests = [],
+ resources = [],
+ data = [],
+ flaky_tests = [],
+ tags = [],
+ prefix = "",
+ jvm_flags = [],
+ args = [],
+ visibility = None,
+ shard_count = 1):
+ for test in _get_test_names(test_files):
+ if test in exclude_tests:
+ continue
+ test_size = default_test_size
+ if test in small_tests:
+ test_size = "small"
+ if test in medium_tests:
+ test_size = "medium"
+ if test in large_tests:
+ test_size = "large"
+ if test in enormous_tests:
+ test_size = "enormous"
+ flaky = 0
+ if (test in flaky_tests) or ("flaky" in tags):
+ flaky = 1
+ java_class = _package_from_path(
+ native.package_name() + "/" + _strip_right(test, ".java"),
+ )
+ package = java_class[:java_class.rfind(".")]
+ native.java_test(
+ name = prefix + test,
+ runtime_deps = deps,
+ resources = resources,
+ size = test_size,
+ jvm_flags = jvm_flags,
+ args = args,
+ flaky = flaky,
+ tags = tags,
+ test_class = java_class,
+ visibility = visibility,
+ shard_count = shard_count,
+ data = data,
+ )
+
+def _get_test_names(test_files):
+ test_names = []
+ for test_file in test_files:
+ if not test_file.endswith("Test.java") and not test_file.endswith("IT.java"):
+ continue
+ test_names += [test_file[:-5]]
+ return test_names
+
+def _package_from_path(package_path, src_impls = None):
+ src_impls = src_impls or ["javatests/", "java/"]
+ for src_impl in src_impls:
+ if not src_impl.endswith("/"):
+ src_impl += "/"
+ index = _index_of_end(package_path, src_impl)
+ if index >= 0:
+ package_path = package_path[index:]
+ break
+ return package_path.replace("/", ".")
+
+def _strip_right(str, suffix):
+ """Returns str without the suffix if it ends with suffix."""
+ if str.endswith(suffix):
+ return str[0:len(str) - len(suffix)]
+ else:
+ return str
+
+def _index_of_end(str, part):
+ """If part is in str, return the index of the first character after part.
+ Return -1 if part is not in str."""
+ index = str.find(part)
+ if index >= 0:
+ return index + len(part)
+ return -1
diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel
new file mode 100644
index 00000000000..ab413d3d060
--- /dev/null
+++ b/broker/BUILD.bazel
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+load("//bazel:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+ name = "broker",
+ srcs = glob(["src/main/java/**/*.java"]),
+ visibility = ["//visibility:public"],
+ deps = [
+ "//acl",
+ "//client",
+ "//common",
+ "//filter",
+ "//remoting",
+ "//srvutil",
+ "//store",
+ "@maven//:ch_qos_logback_logback_classic",
+ "@maven//:com_alibaba_fastjson",
+ "@maven//:com_github_luben_zstd_jni",
+ "@maven//:com_google_guava_guava",
+ "@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru",
+ "@maven//:commons_cli_commons_cli",
+ "@maven//:commons_collections_commons_collections",
+ "@maven//:commons_io_commons_io",
+ "@maven//:commons_validator_commons_validator",
+ "@maven//:io_netty_netty_all",
+ "@maven//:io_openmessaging_storage_dledger",
+ "@maven//:io_opentelemetry_opentelemetry_api",
+ "@maven//:io_opentelemetry_opentelemetry_context",
+ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp",
+ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus",
+ "@maven//:io_opentelemetry_opentelemetry_exporter_logging",
+ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
+ "@maven//:io_opentelemetry_opentelemetry_sdk",
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
+ "@maven//:org_apache_commons_commons_lang3",
+ "@maven//:org_lz4_lz4_java",
+ "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
+ "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
+ "@maven//:org_slf4j_jul_to_slf4j",
+ "@maven//:io_github_aliyunmq_rocketmq_shaded_slf4j_api_bridge",
+ "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
+ "@maven//:net_java_dev_jna_jna",
+ ],
+)
+
+java_library(
+ name = "tests",
+ srcs = glob(["src/test/java/**/*.java"]),
+ resources = [
+ "src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator",
+ "src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener",
+ "src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService",
+ "src/test/resources/rmq.logback-test.xml",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":broker",
+ "//:test_deps",
+ "//acl",
+ "//client",
+ "//common",
+ "//filter",
+ "//remoting",
+ "//store",
+ "@maven//:com_alibaba_fastjson",
+ "@maven//:com_google_guava_guava",
+ "@maven//:io_netty_netty_all",
+ "@maven//:org_apache_commons_commons_lang3",
+ "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
+ "@maven//:org_powermock_powermock_core",
+ "@maven//:io_opentelemetry_opentelemetry_api",
+ ],
+)
+
+GenTestRules(
+ name = "GeneratedTestRules",
+ test_files = glob(["src/test/java/**/*Test.java"]),
+ deps = [
+ ":tests",
+ ],
+)
diff --git a/broker/pom.xml b/broker/pom.xml
index d655959a9a8..3eef8846b78 100644
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@ -1,25 +1,19 @@
-
+
org.apache.rocketmq
rocketmq-all
- 4.2.0
+ 5.2.0
4.0.0
@@ -27,10 +21,14 @@
rocketmq-broker
rocketmq-broker ${project.version}
+
+ ${basedir}/..
+
+
${project.groupId}
- rocketmq-common
+ rocketmq-remoting
${project.groupId}
@@ -38,7 +36,15 @@
${project.groupId}
- rocketmq-remoting
+ rocketmq-tiered-store
+
+
+ io.github.aliyunmq
+ rocketmq-slf4j-api
+
+
+ io.github.aliyunmq
+ rocketmq-logback-classic
${project.groupId}
@@ -53,12 +59,12 @@
rocketmq-filter
- ch.qos.logback
- logback-classic
+ ${project.groupId}
+ rocketmq-acl
- ch.qos.logback
- logback-core
+ commons-io
+ commons-io
com.alibaba
@@ -68,13 +74,29 @@
org.javassist
javassist
+
+ org.bouncycastle
+ bcpkix-jdk15on
+
+
+ com.googlecode.concurrentlinkedhashmap
+ concurrentlinkedhashmap-lru
+
+
+ io.github.aliyunmq
+ rocketmq-shaded-slf4j-api-bridge
+
+
+ org.slf4j
+ jul-to-slf4j
+
maven-surefire-plugin
- 2.19.1
+ ${maven-surefire-plugin.version}
1
false
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
index 0a6f0b45e9d..af90e5f87eb 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
@@ -18,18 +18,33 @@
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.AbstractMap;
import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+
+import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.acl.plain.PlainAccessValidator;
import org.apache.rocketmq.broker.client.ClientHousekeepingService;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ConsumerManager;
@@ -37,105 +52,251 @@
import org.apache.rocketmq.broker.client.ProducerManager;
import org.apache.rocketmq.broker.client.net.Broker2Client;
import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
+import org.apache.rocketmq.broker.coldctr.ColdDataCgCtrService;
+import org.apache.rocketmq.broker.coldctr.ColdDataPullRequestHoldService;
+import org.apache.rocketmq.broker.controller.ReplicasManager;
+import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;
+import org.apache.rocketmq.broker.failover.EscapeBridge;
import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
-import org.apache.rocketmq.broker.filtersrv.FilterServerManager;
import org.apache.rocketmq.broker.latency.BrokerFastFailure;
-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
+import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService;
import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener;
import org.apache.rocketmq.broker.longpolling.PullRequestHoldService;
+import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
+import org.apache.rocketmq.broker.offset.BroadcastOffsetManager;
import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager;
+import org.apache.rocketmq.broker.offset.LmqConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.RocksDBConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.RocksDBLmqConsumerOffsetManager;
import org.apache.rocketmq.broker.out.BrokerOuterAPI;
-import org.apache.rocketmq.broker.plugin.MessageStoreFactory;
-import org.apache.rocketmq.broker.plugin.MessageStorePluginContext;
+import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+import org.apache.rocketmq.broker.processor.AckMessageProcessor;
import org.apache.rocketmq.broker.processor.AdminBrokerProcessor;
+import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor;
import org.apache.rocketmq.broker.processor.ClientManageProcessor;
import org.apache.rocketmq.broker.processor.ConsumerManageProcessor;
import org.apache.rocketmq.broker.processor.EndTransactionProcessor;
+import org.apache.rocketmq.broker.processor.NotificationProcessor;
+import org.apache.rocketmq.broker.processor.PeekMessageProcessor;
+import org.apache.rocketmq.broker.processor.PollingInfoProcessor;
+import org.apache.rocketmq.broker.processor.PopInflightMessageCounter;
+import org.apache.rocketmq.broker.processor.PopMessageProcessor;
import org.apache.rocketmq.broker.processor.PullMessageProcessor;
+import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;
import org.apache.rocketmq.broker.processor.QueryMessageProcessor;
+import org.apache.rocketmq.broker.processor.ReplyMessageProcessor;
import org.apache.rocketmq.broker.processor.SendMessageProcessor;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
import org.apache.rocketmq.broker.slave.SlaveSynchronize;
+import org.apache.rocketmq.broker.subscription.LmqSubscriptionGroupManager;
+import org.apache.rocketmq.broker.subscription.RocksDBLmqSubscriptionGroupManager;
+import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager;
import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.LmqTopicConfigManager;
+import org.apache.rocketmq.broker.topic.RocksDBLmqTopicConfigManager;
+import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager;
import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService;
+import org.apache.rocketmq.broker.topic.TopicQueueMappingManager;
+import org.apache.rocketmq.broker.topic.TopicRouteInfoManager;
+import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
+import org.apache.rocketmq.broker.transaction.TransactionMetricsFlushService;
+import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService;
+import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
+import org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageCheckListener;
+import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge;
+import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl;
+import org.apache.rocketmq.broker.util.HookUtils;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
import org.apache.rocketmq.common.BrokerConfig;
-import org.apache.rocketmq.common.Configuration;
+import org.apache.rocketmq.common.BrokerIdentity;
+import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
-import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.stats.MomentStatsItem;
+import org.apache.rocketmq.common.utils.ServiceProvider;
+import org.apache.rocketmq.common.utils.ThreadUtils;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.Configuration;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.RemotingServer;
+import org.apache.rocketmq.remoting.common.TlsMode;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
+import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
+import org.apache.rocketmq.srvutil.FileWatchService;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.MessageArrivingListener;
import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.RocksDBMessageStore;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
+import org.apache.rocketmq.store.hook.PutMessageHook;
+import org.apache.rocketmq.store.hook.SendMessageBackHook;
+import org.apache.rocketmq.store.plugin.MessageStoreFactory;
+import org.apache.rocketmq.store.plugin.MessageStorePluginContext;
import org.apache.rocketmq.store.stats.BrokerStats;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.rocketmq.store.stats.LmqBrokerStatsManager;
+import org.apache.rocketmq.store.timer.TimerCheckpoint;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.apache.rocketmq.store.timer.TimerMetrics;
public class BrokerController {
- private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME);
private static final Logger LOG_WATER_MARK = LoggerFactory.getLogger(LoggerName.WATER_MARK_LOGGER_NAME);
- private final BrokerConfig brokerConfig;
+ protected static final int HA_ADDRESS_MIN_LENGTH = 6;
+
+ protected final BrokerConfig brokerConfig;
private final NettyServerConfig nettyServerConfig;
private final NettyClientConfig nettyClientConfig;
- private final MessageStoreConfig messageStoreConfig;
- private final ConsumerOffsetManager consumerOffsetManager;
- private final ConsumerManager consumerManager;
- private final ConsumerFilterManager consumerFilterManager;
- private final ProducerManager producerManager;
- private final ClientHousekeepingService clientHousekeepingService;
- private final PullMessageProcessor pullMessageProcessor;
- private final PullRequestHoldService pullRequestHoldService;
- private final MessageArrivingListener messageArrivingListener;
- private final Broker2Client broker2Client;
- private final SubscriptionGroupManager subscriptionGroupManager;
- private final ConsumerIdsChangeListener consumerIdsChangeListener;
+ protected final MessageStoreConfig messageStoreConfig;
+ protected final ConsumerOffsetManager consumerOffsetManager;
+ protected final BroadcastOffsetManager broadcastOffsetManager;
+ protected final ConsumerManager consumerManager;
+ protected final ConsumerFilterManager consumerFilterManager;
+ protected final ConsumerOrderInfoManager consumerOrderInfoManager;
+ protected final PopInflightMessageCounter popInflightMessageCounter;
+ protected final ProducerManager producerManager;
+ protected final ScheduleMessageService scheduleMessageService;
+ protected final ClientHousekeepingService clientHousekeepingService;
+ protected final PullMessageProcessor pullMessageProcessor;
+ protected final PeekMessageProcessor peekMessageProcessor;
+ protected final PopMessageProcessor popMessageProcessor;
+ protected final AckMessageProcessor ackMessageProcessor;
+ protected final ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor;
+ protected final NotificationProcessor notificationProcessor;
+ protected final PollingInfoProcessor pollingInfoProcessor;
+ protected final QueryAssignmentProcessor queryAssignmentProcessor;
+ protected final ClientManageProcessor clientManageProcessor;
+ protected final SendMessageProcessor sendMessageProcessor;
+ protected final ReplyMessageProcessor replyMessageProcessor;
+ protected final PullRequestHoldService pullRequestHoldService;
+ protected final MessageArrivingListener messageArrivingListener;
+ protected final Broker2Client broker2Client;
+ protected final ConsumerIdsChangeListener consumerIdsChangeListener;
+ protected final EndTransactionProcessor endTransactionProcessor;
private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager();
- private final BrokerOuterAPI brokerOuterAPI;
- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(
- "BrokerControllerScheduledThread"));
- private final SlaveSynchronize slaveSynchronize;
- private final BlockingQueue sendThreadPoolQueue;
- private final BlockingQueue pullThreadPoolQueue;
- private final BlockingQueue queryThreadPoolQueue;
- private final BlockingQueue clientManagerThreadPoolQueue;
- private final BlockingQueue consumerManagerThreadPoolQueue;
- private final FilterServerManager filterServerManager;
- private final BrokerStatsManager brokerStatsManager;
- private final List sendMessageHookList = new ArrayList();
- private final List consumeMessageHookList = new ArrayList();
- private MessageStore messageStore;
- private RemotingServer remotingServer;
- private RemotingServer fastRemotingServer;
- private TopicConfigManager topicConfigManager;
- private ExecutorService sendMessageExecutor;
- private ExecutorService pullMessageExecutor;
- private ExecutorService queryMessageExecutor;
- private ExecutorService adminBrokerExecutor;
- private ExecutorService clientManageExecutor;
- private ExecutorService consumerManageExecutor;
- private boolean updateMasterHAServerAddrPeriodically = false;
+ private final TopicRouteInfoManager topicRouteInfoManager;
+ protected BrokerOuterAPI brokerOuterAPI;
+ protected ScheduledExecutorService scheduledExecutorService;
+ protected ScheduledExecutorService syncBrokerMemberGroupExecutorService;
+ protected ScheduledExecutorService brokerHeartbeatExecutorService;
+ protected final SlaveSynchronize slaveSynchronize;
+ protected final BlockingQueue sendThreadPoolQueue;
+ protected final BlockingQueue putThreadPoolQueue;
+ protected final BlockingQueue ackThreadPoolQueue;
+ protected final BlockingQueue pullThreadPoolQueue;
+ protected final BlockingQueue litePullThreadPoolQueue;
+ protected final BlockingQueue replyThreadPoolQueue;
+ protected final BlockingQueue queryThreadPoolQueue;
+ protected final BlockingQueue clientManagerThreadPoolQueue;
+ protected final BlockingQueue heartbeatThreadPoolQueue;
+ protected final BlockingQueue consumerManagerThreadPoolQueue;
+ protected final BlockingQueue endTransactionThreadPoolQueue;
+ protected final BlockingQueue adminBrokerThreadPoolQueue;
+ protected final BlockingQueue loadBalanceThreadPoolQueue;
+ protected final BrokerStatsManager brokerStatsManager;
+ protected final List sendMessageHookList = new ArrayList<>();
+ protected final List consumeMessageHookList = new ArrayList<>();
+ protected MessageStore messageStore;
+ protected RemotingServer remotingServer;
+ protected CountDownLatch remotingServerStartLatch;
+ protected RemotingServer fastRemotingServer;
+ protected TopicConfigManager topicConfigManager;
+ protected SubscriptionGroupManager subscriptionGroupManager;
+ protected TopicQueueMappingManager topicQueueMappingManager;
+ protected ExecutorService sendMessageExecutor;
+ protected ExecutorService pullMessageExecutor;
+ protected ExecutorService litePullMessageExecutor;
+ protected ExecutorService putMessageFutureExecutor;
+ protected ExecutorService ackMessageExecutor;
+ protected ExecutorService replyMessageExecutor;
+ protected ExecutorService queryMessageExecutor;
+ protected ExecutorService adminBrokerExecutor;
+ protected ExecutorService clientManageExecutor;
+ protected ExecutorService heartbeatExecutor;
+ protected ExecutorService consumerManageExecutor;
+ protected ExecutorService loadBalanceExecutor;
+ protected ExecutorService endTransactionExecutor;
+ protected boolean updateMasterHAServerAddrPeriodically = false;
private BrokerStats brokerStats;
private InetSocketAddress storeHost;
- private BrokerFastFailure brokerFastFailure;
+ private TimerMessageStore timerMessageStore;
+ private TimerCheckpoint timerCheckpoint;
+ protected BrokerFastFailure brokerFastFailure;
private Configuration configuration;
+ protected TopicQueueMappingCleanService topicQueueMappingCleanService;
+ protected FileWatchService fileWatchService;
+ protected TransactionalMessageCheckService transactionalMessageCheckService;
+ protected TransactionalMessageService transactionalMessageService;
+ protected AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;
+ protected Map accessValidatorMap = new HashMap<>();
+ protected volatile boolean shutdown = false;
+ protected ShutdownHook shutdownHook;
+ private volatile boolean isScheduleServiceStart = false;
+ private volatile boolean isTransactionCheckServiceStart = false;
+ protected volatile BrokerMemberGroup brokerMemberGroup;
+ protected EscapeBridge escapeBridge;
+ protected List brokerAttachedPlugins = new ArrayList<>();
+ protected volatile long shouldStartTime;
+ private BrokerPreOnlineService brokerPreOnlineService;
+ protected volatile boolean isIsolated = false;
+ protected volatile long minBrokerIdInGroup = 0;
+ protected volatile String minBrokerAddrInGroup = null;
+ private final Lock lock = new ReentrantLock();
+ protected final List> scheduledFutures = new ArrayList<>();
+ protected ReplicasManager replicasManager;
+ private long lastSyncTimeMs = System.currentTimeMillis();
+ private BrokerMetricsManager brokerMetricsManager;
+ private ColdDataPullRequestHoldService coldDataPullRequestHoldService;
+ private ColdDataCgCtrService coldDataCgCtrService;
+ private TransactionMetricsFlushService transactionMetricsFlushService;
+
+ public BrokerController(
+ final BrokerConfig brokerConfig,
+ final NettyServerConfig nettyServerConfig,
+ final NettyClientConfig nettyClientConfig,
+ final MessageStoreConfig messageStoreConfig,
+ final ShutdownHook shutdownHook
+ ) {
+ this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig);
+ this.shutdownHook = shutdownHook;
+ }
+
+ public BrokerController(
+ final BrokerConfig brokerConfig,
+ final MessageStoreConfig messageStoreConfig
+ ) {
+ this(brokerConfig, null, null, messageStoreConfig);
+ }
public BrokerController(
final BrokerConfig brokerConfig,
@@ -147,38 +308,112 @@ public BrokerController(
this.nettyServerConfig = nettyServerConfig;
this.nettyClientConfig = nettyClientConfig;
this.messageStoreConfig = messageStoreConfig;
- this.consumerOffsetManager = new ConsumerOffsetManager(this);
- this.topicConfigManager = new TopicConfigManager(this);
+ this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort()));
+ this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat());
+ this.broadcastOffsetManager = new BroadcastOffsetManager(this);
+ if (this.messageStoreConfig.isEnableRocksDBStore()) {
+ this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this);
+ this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this);
+ this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this);
+ } else {
+ this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new LmqTopicConfigManager(this) : new TopicConfigManager(this);
+ this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new LmqSubscriptionGroupManager(this) : new SubscriptionGroupManager(this);
+ this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new LmqConsumerOffsetManager(this) : new ConsumerOffsetManager(this);
+ }
+ this.topicQueueMappingManager = new TopicQueueMappingManager(this);
this.pullMessageProcessor = new PullMessageProcessor(this);
- this.pullRequestHoldService = new PullRequestHoldService(this);
- this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService);
+ this.peekMessageProcessor = new PeekMessageProcessor(this);
+ this.pullRequestHoldService = messageStoreConfig.isEnableLmq() ? new LmqPullRequestHoldService(this) : new PullRequestHoldService(this);
+ this.popMessageProcessor = new PopMessageProcessor(this);
+ this.notificationProcessor = new NotificationProcessor(this);
+ this.pollingInfoProcessor = new PollingInfoProcessor(this);
+ this.ackMessageProcessor = new AckMessageProcessor(this);
+ this.changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(this);
+ this.sendMessageProcessor = new SendMessageProcessor(this);
+ this.replyMessageProcessor = new ReplyMessageProcessor(this);
+ this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService, this.popMessageProcessor, this.notificationProcessor);
this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this);
- this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener);
+ this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener, this.brokerStatsManager, this.brokerConfig);
+ this.producerManager = new ProducerManager(this.brokerStatsManager);
this.consumerFilterManager = new ConsumerFilterManager(this);
- this.producerManager = new ProducerManager();
+ this.consumerOrderInfoManager = new ConsumerOrderInfoManager(this);
+ this.popInflightMessageCounter = new PopInflightMessageCounter(this);
this.clientHousekeepingService = new ClientHousekeepingService(this);
this.broker2Client = new Broker2Client(this);
- this.subscriptionGroupManager = new SubscriptionGroupManager(this);
- this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
- this.filterServerManager = new FilterServerManager(this);
-
- this.slaveSynchronize = new SlaveSynchronize(this);
+ this.scheduleMessageService = new ScheduleMessageService(this);
+ this.coldDataPullRequestHoldService = new ColdDataPullRequestHoldService(this);
+ this.coldDataCgCtrService = new ColdDataCgCtrService(this);
- this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity());
- this.pullThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity());
- this.queryThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getQueryThreadPoolQueueCapacity());
- this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity());
- this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity());
+ if (nettyClientConfig != null) {
+ this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
+ }
- this.brokerStatsManager = new BrokerStatsManager(this.brokerConfig.getBrokerClusterName());
- this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort()));
+ this.queryAssignmentProcessor = new QueryAssignmentProcessor(this);
+ this.clientManageProcessor = new ClientManageProcessor(this);
+ this.slaveSynchronize = new SlaveSynchronize(this);
+ this.endTransactionProcessor = new EndTransactionProcessor(this);
+
+ this.sendThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getSendThreadPoolQueueCapacity());
+ this.putThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getPutThreadPoolQueueCapacity());
+ this.pullThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getPullThreadPoolQueueCapacity());
+ this.litePullThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getLitePullThreadPoolQueueCapacity());
+
+ this.ackThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getAckThreadPoolQueueCapacity());
+ this.replyThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getReplyThreadPoolQueueCapacity());
+ this.queryThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getQueryThreadPoolQueueCapacity());
+ this.clientManagerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getClientManagerThreadPoolQueueCapacity());
+ this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity());
+ this.heartbeatThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getHeartbeatThreadPoolQueueCapacity());
+ this.endTransactionThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getEndTransactionPoolQueueCapacity());
+ this.adminBrokerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getAdminBrokerThreadPoolQueueCapacity());
+ this.loadBalanceThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getLoadBalanceThreadPoolQueueCapacity());
this.brokerFastFailure = new BrokerFastFailure(this);
+
+ String brokerConfigPath;
+ if (brokerConfig.getBrokerConfigPath() != null && !brokerConfig.getBrokerConfigPath().isEmpty()) {
+ brokerConfigPath = brokerConfig.getBrokerConfigPath();
+ } else {
+ brokerConfigPath = BrokerPathConfigHelper.getBrokerConfigPath();
+ }
this.configuration = new Configuration(
- log,
- BrokerPathConfigHelper.getBrokerConfigPath(),
+ LOG,
+ brokerConfigPath,
this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig
);
+
+ this.brokerStatsManager.setProduerStateGetter(new BrokerStatsManager.StateGetter() {
+ @Override
+ public boolean online(String instanceId, String group, String topic) {
+ if (getTopicConfigManager().getTopicConfigTable().containsKey(NamespaceUtil.wrapNamespace(instanceId, topic))) {
+ return getProducerManager().groupOnline(NamespaceUtil.wrapNamespace(instanceId, group));
+ } else {
+ return getProducerManager().groupOnline(group);
+ }
+ }
+ });
+ this.brokerStatsManager.setConsumerStateGetter(new BrokerStatsManager.StateGetter() {
+ @Override
+ public boolean online(String instanceId, String group, String topic) {
+ String topicFullName = NamespaceUtil.wrapNamespace(instanceId, topic);
+ if (getTopicConfigManager().getTopicConfigTable().containsKey(topicFullName)) {
+ return getConsumerManager().findSubscriptionData(NamespaceUtil.wrapNamespace(instanceId, group), topicFullName) != null;
+ } else {
+ return getConsumerManager().findSubscriptionData(group, topic) != null;
+ }
+ }
+ });
+
+ this.brokerMemberGroup = new BrokerMemberGroup(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName());
+ this.brokerMemberGroup.getBrokerAddrs().put(this.brokerConfig.getBrokerId(), this.getBrokerAddr());
+
+ this.escapeBridge = new EscapeBridge(this);
+
+ this.topicRouteInfoManager = new TopicRouteInfoManager(this);
+
+ if (this.brokerConfig.isEnableSlaveActingMaster() && !this.brokerConfig.isSkipPreOnline()) {
+ this.brokerPreOnlineService = new BrokerPreOnlineService(this);
+ }
}
public BrokerConfig getBrokerConfig() {
@@ -189,6 +424,10 @@ public NettyServerConfig getNettyServerConfig() {
return nettyServerConfig;
}
+ public NettyClientConfig getNettyClientConfig() {
+ return nettyClientConfig;
+ }
+
public BlockingQueue getPullThreadPoolQueue() {
return pullThreadPoolQueue;
}
@@ -197,166 +436,220 @@ public BlockingQueue getQueryThreadPoolQueue() {
return queryThreadPoolQueue;
}
- public boolean initialize() throws CloneNotSupportedException {
- boolean result = this.topicConfigManager.load();
+ public BrokerMetricsManager getBrokerMetricsManager() {
+ return brokerMetricsManager;
+ }
- result = result && this.consumerOffsetManager.load();
- result = result && this.subscriptionGroupManager.load();
- result = result && this.consumerFilterManager.load();
+ protected void initializeRemotingServer() throws CloneNotSupportedException {
+ this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
+ NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
- if (result) {
- try {
- this.messageStore =
- new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener,
- this.brokerConfig);
- this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
- //load plugin
- MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
- this.messageStore = MessageStoreFactory.build(context, this.messageStore);
- this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
- } catch (IOException e) {
- result = false;
- log.error("Failed to initialize", e);
- }
+ int listeningPort = nettyServerConfig.getListenPort() - 2;
+ if (listeningPort < 0) {
+ listeningPort = 0;
}
+ fastConfig.setListenPort(listeningPort);
- result = result && this.messageStore.load();
+ this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
+ }
- if (result) {
- this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
- NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
- fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
- this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
- this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(
- this.brokerConfig.getSendMessageThreadPoolNums(),
- this.brokerConfig.getSendMessageThreadPoolNums(),
- 1000 * 60,
- TimeUnit.MILLISECONDS,
- this.sendThreadPoolQueue,
- new ThreadFactoryImpl("SendMessageThread_"));
-
- this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(
- this.brokerConfig.getPullMessageThreadPoolNums(),
- this.brokerConfig.getPullMessageThreadPoolNums(),
- 1000 * 60,
- TimeUnit.MILLISECONDS,
- this.pullThreadPoolQueue,
- new ThreadFactoryImpl("PullMessageThread_"));
-
- this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor(
- this.brokerConfig.getQueryMessageThreadPoolNums(),
- this.brokerConfig.getQueryMessageThreadPoolNums(),
- 1000 * 60,
- TimeUnit.MILLISECONDS,
- this.queryThreadPoolQueue,
- new ThreadFactoryImpl("QueryMessageThread_"));
-
- this.adminBrokerExecutor =
- Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl(
- "AdminBrokerThread_"));
-
- this.clientManageExecutor = new ThreadPoolExecutor(
- this.brokerConfig.getClientManageThreadPoolNums(),
- this.brokerConfig.getClientManageThreadPoolNums(),
- 1000 * 60,
- TimeUnit.MILLISECONDS,
- this.clientManagerThreadPoolQueue,
- new ThreadFactoryImpl("ClientManageThread_"));
-
- this.consumerManageExecutor =
- Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl(
- "ConsumerManageThread_"));
-
- this.registerProcessor();
-
- final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis();
- final long period = 1000 * 60 * 60 * 24;
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.getBrokerStats().record();
- } catch (Throwable e) {
- log.error("schedule record error.", e);
- }
- }
- }, initialDelay, period, TimeUnit.MILLISECONDS);
+ /**
+ * Initialize resources including remoting server and thread executors.
+ */
+ protected void initializeResources() {
+ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
+ new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity()));
+
+ this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getSendMessageThreadPoolNums(),
+ this.brokerConfig.getSendMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.sendThreadPoolQueue,
+ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity()));
+
+ this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getPullMessageThreadPoolNums(),
+ this.brokerConfig.getPullMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.pullThreadPoolQueue,
+ new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity()));
+
+ this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getLitePullMessageThreadPoolNums(),
+ this.brokerConfig.getLitePullMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.litePullThreadPoolQueue,
+ new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity()));
+
+ this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getPutMessageFutureThreadPoolNums(),
+ this.brokerConfig.getPutMessageFutureThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.putThreadPoolQueue,
+ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity()));
+
+ this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getAckMessageThreadPoolNums(),
+ this.brokerConfig.getAckMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.ackThreadPoolQueue,
+ new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity()));
+
+ this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getQueryMessageThreadPoolNums(),
+ this.brokerConfig.getQueryMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.queryThreadPoolQueue,
+ new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity()));
+
+ this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getAdminBrokerThreadPoolNums(),
+ this.brokerConfig.getAdminBrokerThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.adminBrokerThreadPoolQueue,
+ new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity()));
+
+ this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getClientManageThreadPoolNums(),
+ this.brokerConfig.getClientManageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.clientManagerThreadPoolQueue,
+ new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity()));
+
+ this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getHeartbeatThreadPoolNums(),
+ this.brokerConfig.getHeartbeatThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.heartbeatThreadPoolQueue,
+ new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity()));
+
+ this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getConsumerManageThreadPoolNums(),
+ this.brokerConfig.getConsumerManageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.consumerManagerThreadPoolQueue,
+ new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity()));
+
+ this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
+ this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.replyThreadPoolQueue,
+ new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity()));
+
+ this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getEndTransactionThreadPoolNums(),
+ this.brokerConfig.getEndTransactionThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.endTransactionThreadPoolQueue,
+ new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity()));
+
+ this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor(
+ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),
+ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ this.loadBalanceThreadPoolQueue,
+ new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity()));
+
+ this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1,
+ new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity()));
+ this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1,
+ new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity()));
+
+ this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this);
+ }
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.consumerOffsetManager.persist();
- } catch (Throwable e) {
- log.error("schedule persist consumerOffset error.", e);
- }
+ protected void initializeBrokerScheduledTasks() {
+ final long initialDelay = UtilAll.computeNextMorningTimeMillis() - System.currentTimeMillis();
+ final long period = TimeUnit.DAYS.toMillis(1);
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.getBrokerStats().record();
+ } catch (Throwable e) {
+ LOG.error("BrokerController: failed to record broker stats", e);
}
- }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
+ }
+ }, initialDelay, period, TimeUnit.MILLISECONDS);
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.consumerFilterManager.persist();
- } catch (Throwable e) {
- log.error("schedule persist consumer filter error.", e);
- }
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.consumerOffsetManager.persist();
+ } catch (Throwable e) {
+ LOG.error(
+ "BrokerController: failed to persist config file of consumerOffset", e);
}
- }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
+ }
+ }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.protectBroker();
- } catch (Throwable e) {
- log.error("protectBroker error.", e);
- }
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.consumerFilterManager.persist();
+ BrokerController.this.consumerOrderInfoManager.persist();
+ } catch (Throwable e) {
+ LOG.error(
+ "BrokerController: failed to persist config file of consumerFilter or consumerOrderInfo",
+ e);
}
- }, 3, 3, TimeUnit.MINUTES);
+ }
+ }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.printWaterMark();
- } catch (Throwable e) {
- log.error("printWaterMark error.", e);
- }
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.protectBroker();
+ } catch (Throwable e) {
+ LOG.error("BrokerController: failed to protectBroker", e);
}
- }, 10, 1, TimeUnit.SECONDS);
-
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ }
+ }, 3, 3, TimeUnit.MINUTES);
- @Override
- public void run() {
- try {
- log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes());
- } catch (Throwable e) {
- log.error("schedule dispatchBehindBytes error.", e);
- }
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.printWaterMark();
+ } catch (Throwable e) {
+ LOG.error("BrokerController: failed to print broker watermark", e);
}
- }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
+ }
+ }, 10, 1, TimeUnit.SECONDS);
- if (this.brokerConfig.getNamesrvAddr() != null) {
- this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
- log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
- } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
- } catch (Throwable e) {
- log.error("ScheduledTask fetchNameServerAddr exception", e);
- }
- }
- }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
+ @Override
+ public void run() {
+ try {
+ LOG.info("Dispatch task fall behind commit log {}bytes",
+ BrokerController.this.getMessageStore().dispatchBehindBytes());
+ } catch (Throwable e) {
+ LOG.error("Failed to print dispatchBehindBytes", e);
+ }
}
+ }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
+ if (!messageStoreConfig.isEnableDLegerCommitLog() && !messageStoreConfig.isDuplicationEnable() && !brokerConfig.isEnableControllerMode()) {
if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
- if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
+ if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= HA_ADDRESS_MIN_LENGTH) {
this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
this.updateMasterHAServerAddrPeriodically = false;
} else {
@@ -368,12 +661,21 @@ public void run() {
@Override
public void run() {
try {
- BrokerController.this.slaveSynchronize.syncAll();
+ if (System.currentTimeMillis() - lastSyncTimeMs > 60 * 1000) {
+ BrokerController.this.getSlaveSynchronize().syncAll();
+ lastSyncTimeMs = System.currentTimeMillis();
+ }
+
+ //timer checkpoint, latency-sensitive, so sync it more frequently
+ if (messageStoreConfig.isTimerWheelEnable()) {
+ BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint();
+ }
} catch (Throwable e) {
- log.error("ScheduledTask syncAll slave exception", e);
+ LOG.error("Failed to sync all config for slave.", e);
}
}
- }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
+ }, 1000 * 10, 3 * 1000, TimeUnit.MILLISECONDS);
+
} else {
this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@@ -382,155 +684,531 @@ public void run() {
try {
BrokerController.this.printMasterAndSlaveDiff();
} catch (Throwable e) {
- log.error("schedule printMasterAndSlaveDiff error.", e);
+ LOG.error("Failed to print diff of master and slave.", e);
}
}
}, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
}
}
- return result;
+ if (this.brokerConfig.isEnableControllerMode()) {
+ this.updateMasterHAServerAddrPeriodically = true;
+ }
}
- public void registerProcessor() {
- /**
- * SendMessageProcessor
- */
- SendMessageProcessor sendProcessor = new SendMessageProcessor(this);
- sendProcessor.registerSendMessageHook(sendMessageHookList);
- sendProcessor.registerConsumeMessageHook(consumeMessageHookList);
-
- this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor);
- this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor);
- this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor);
- this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor);
- /**
- * PullMessageProcessor
- */
- this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);
- this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
-
- /**
- * QueryMessageProcessor
- */
- NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this);
- this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
- this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
-
- this.fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
-
- /**
- * ClientManageProcessor
- */
- ClientManageProcessor clientProcessor = new ClientManageProcessor(this);
- this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor);
- this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
- this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
-
- this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
+ protected void initializeScheduledTasks() {
- /**
- * ConsumerManageProcessor
- */
- ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this);
- this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
- this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
- this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
-
- this.fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+ initializeBrokerScheduledTasks();
- /**
- * EndTransactionProcessor
- */
- this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.sendMessageExecutor);
- this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.sendMessageExecutor);
+ if (this.brokerConfig.getNamesrvAddr() != null) {
+ this.updateNamesrvAddr();
+ LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
+ // also auto update namesrv if specify
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.updateNamesrvAddr();
+ } catch (Throwable e) {
+ LOG.error("Failed to update nameServer address list", e);
+ }
+ }
+ }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
+ } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
- /**
- * Default
- */
- AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this);
- this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
- this.fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
+ } catch (Throwable e) {
+ LOG.error("Failed to fetch nameServer address", e);
+ }
+ }
+ }, 1000 * 10, this.brokerConfig.getFetchNamesrvAddrInterval(), TimeUnit.MILLISECONDS);
+ }
}
- public BrokerStats getBrokerStats() {
- return brokerStats;
+ private void updateNamesrvAddr() {
+ if (this.brokerConfig.isFetchNameSrvAddrByDnsLookup()) {
+ this.brokerOuterAPI.updateNameServerAddressListByDnsLookup(this.brokerConfig.getNamesrvAddr());
+ } else {
+ this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
+ }
}
- public void setBrokerStats(BrokerStats brokerStats) {
- this.brokerStats = brokerStats;
+ public boolean initializeMetadata() {
+ boolean result = this.topicConfigManager.load();
+ result = result && this.topicQueueMappingManager.load();
+ result = result && this.consumerOffsetManager.load();
+ result = result && this.subscriptionGroupManager.load();
+ result = result && this.consumerFilterManager.load();
+ result = result && this.consumerOrderInfoManager.load();
+ return result;
}
- public void protectBroker() {
- if (this.brokerConfig.isDisableConsumeIfConsumerReadSlowly()) {
- final Iterator> it = this.brokerStatsManager.getMomentStatsItemSetFallSize().getStatsItemTable().entrySet().iterator();
- while (it.hasNext()) {
- final Map.Entry next = it.next();
- final long fallBehindBytes = next.getValue().getValue().get();
- if (fallBehindBytes > this.brokerConfig.getConsumerFallbehindThreshold()) {
- final String[] split = next.getValue().getStatsKey().split("@");
- final String group = split[2];
- LOG_PROTECTION.info("[PROTECT_BROKER] the consumer[{}] consume slowly, {} bytes, disable it", group, fallBehindBytes);
- this.subscriptionGroupManager.disableConsume(group);
- }
+ public boolean initializeMessageStore() {
+ boolean result = true;
+ try {
+ DefaultMessageStore defaultMessageStore;
+ if (this.messageStoreConfig.isEnableRocksDBStore()) {
+ defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
+ } else {
+ defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
}
+
+ if (messageStoreConfig.isEnableDLegerCommitLog()) {
+ DLedgerRoleChangeHandler roleChangeHandler =
+ new DLedgerRoleChangeHandler(this, defaultMessageStore);
+ ((DLedgerCommitLog) defaultMessageStore.getCommitLog())
+ .getdLedgerServer().getDLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);
+ }
+
+ this.brokerStats = new BrokerStats(defaultMessageStore);
+
+ // Load store plugin
+ MessageStorePluginContext context = new MessageStorePluginContext(
+ messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, configuration);
+ this.messageStore = MessageStoreFactory.build(context, defaultMessageStore);
+ this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
+ if (messageStoreConfig.isTimerWheelEnable()) {
+ this.timerCheckpoint = new TimerCheckpoint(BrokerPathConfigHelper.getTimerCheckPath(messageStoreConfig.getStorePathRootDir()));
+ TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir()));
+ this.timerMessageStore = new TimerMessageStore(messageStore, messageStoreConfig, timerCheckpoint, timerMetrics, brokerStatsManager);
+ this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg));
+ this.messageStore.setTimerMessageStore(this.timerMessageStore);
+ }
+ } catch (IOException e) {
+ result = false;
+ LOG.error("BrokerController#initialize: unexpected error occurs", e);
}
+ return result;
}
- public long headSlowTimeMills(BlockingQueue q) {
- long slowTimeMills = 0;
- final Runnable peek = q.peek();
- if (peek != null) {
- RequestTask rt = BrokerFastFailure.castRunnable(peek);
- slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp();
- }
+ public boolean initialize() throws CloneNotSupportedException {
- if (slowTimeMills < 0)
- slowTimeMills = 0;
+ boolean result = this.initializeMetadata();
+ if (!result) {
+ return false;
+ }
- return slowTimeMills;
- }
+ result = this.initializeMessageStore();
+ if (!result) {
+ return false;
+ }
- public long headSlowTimeMills4SendThreadPoolQueue() {
- return this.headSlowTimeMills(this.sendThreadPoolQueue);
+ return this.recoverAndInitService();
}
- public long headSlowTimeMills4PullThreadPoolQueue() {
- return this.headSlowTimeMills(this.pullThreadPoolQueue);
- }
+ public boolean recoverAndInitService() throws CloneNotSupportedException {
- public long headSlowTimeMills4QueryThreadPoolQueue() {
- return this.headSlowTimeMills(this.queryThreadPoolQueue);
- }
+ boolean result = true;
- public void printWaterMark() {
- LOG_WATER_MARK.info("[WATERMARK] Send Queue Size: {} SlowTimeMills: {}", this.sendThreadPoolQueue.size(), headSlowTimeMills4SendThreadPoolQueue());
- LOG_WATER_MARK.info("[WATERMARK] Pull Queue Size: {} SlowTimeMills: {}", this.pullThreadPoolQueue.size(), headSlowTimeMills4PullThreadPoolQueue());
- LOG_WATER_MARK.info("[WATERMARK] Query Queue Size: {} SlowTimeMills: {}", this.queryThreadPoolQueue.size(), headSlowTimeMills4QueryThreadPoolQueue());
- }
+ if (this.brokerConfig.isEnableControllerMode()) {
+ this.replicasManager = new ReplicasManager(this);
+ this.replicasManager.setFenced(true);
+ }
- public MessageStore getMessageStore() {
- return messageStore;
- }
+ if (messageStore != null) {
+ registerMessageStoreHook();
+ result = this.messageStore.load();
+ }
- public void setMessageStore(MessageStore messageStore) {
- this.messageStore = messageStore;
- }
+ if (messageStoreConfig.isTimerWheelEnable()) {
+ result = result && this.timerMessageStore.load();
+ }
- private void printMasterAndSlaveDiff() {
- long diff = this.messageStore.slaveFallBehindMuch();
+ //scheduleMessageService load after messageStore load success
+ result = result && this.scheduleMessageService.load();
- // XXX: warn and notify me
- log.info("Slave fall behind master: {} bytes", diff);
+ for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
+ if (brokerAttachedPlugin != null) {
+ result = result && brokerAttachedPlugin.load();
+ }
+ }
+
+ this.brokerMetricsManager = new BrokerMetricsManager(this);
+
+ if (result) {
+
+ initializeRemotingServer();
+
+ initializeResources();
+
+ registerProcessor();
+
+ initializeScheduledTasks();
+
+ initialTransaction();
+
+ initialAcl();
+
+ initialRpcHooks();
+
+ if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {
+ // Register a listener to reload SslContext
+ try {
+ fileWatchService = new FileWatchService(
+ new String[] {
+ TlsSystemConfig.tlsServerCertPath,
+ TlsSystemConfig.tlsServerKeyPath,
+ TlsSystemConfig.tlsServerTrustCertPath
+ },
+ new FileWatchService.Listener() {
+ boolean certChanged, keyChanged = false;
+
+ @Override
+ public void onChanged(String path) {
+ if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) {
+ LOG.info("The trust certificate changed, reload the ssl context");
+ reloadServerSslContext();
+ }
+ if (path.equals(TlsSystemConfig.tlsServerCertPath)) {
+ certChanged = true;
+ }
+ if (path.equals(TlsSystemConfig.tlsServerKeyPath)) {
+ keyChanged = true;
+ }
+ if (certChanged && keyChanged) {
+ LOG.info("The certificate and private key changed, reload the ssl context");
+ certChanged = keyChanged = false;
+ reloadServerSslContext();
+ }
+ }
+
+ private void reloadServerSslContext() {
+ ((NettyRemotingServer) remotingServer).loadSslContext();
+ ((NettyRemotingServer) fastRemotingServer).loadSslContext();
+ }
+ });
+ } catch (Exception e) {
+ result = false;
+ LOG.warn("FileWatchService created error, can't load the certificate dynamically");
+ }
+ }
+ }
+
+ return result;
+ }
+
+ public void registerMessageStoreHook() {
+ List putMessageHookList = messageStore.getPutMessageHookList();
+
+ putMessageHookList.add(new PutMessageHook() {
+ @Override
+ public String hookName() {
+ return "checkBeforePutMessage";
+ }
+
+ @Override
+ public PutMessageResult executeBeforePutMessage(MessageExt msg) {
+ return HookUtils.checkBeforePutMessage(BrokerController.this, msg);
+ }
+ });
+
+ putMessageHookList.add(new PutMessageHook() {
+ @Override
+ public String hookName() {
+ return "innerBatchChecker";
+ }
+
+ @Override
+ public PutMessageResult executeBeforePutMessage(MessageExt msg) {
+ if (msg instanceof MessageExtBrokerInner) {
+ return HookUtils.checkInnerBatch(BrokerController.this, msg);
+ }
+ return null;
+ }
+ });
+
+ putMessageHookList.add(new PutMessageHook() {
+ @Override
+ public String hookName() {
+ return "handleScheduleMessage";
+ }
+
+ @Override
+ public PutMessageResult executeBeforePutMessage(MessageExt msg) {
+ if (msg instanceof MessageExtBrokerInner) {
+ return HookUtils.handleScheduleMessage(BrokerController.this, (MessageExtBrokerInner) msg);
+ }
+ return null;
+ }
+ });
+
+ SendMessageBackHook sendMessageBackHook = new SendMessageBackHook() {
+ @Override
+ public boolean executeSendMessageBack(List msgList, String brokerName, String brokerAddr) {
+ return HookUtils.sendMessageBack(BrokerController.this, msgList, brokerName, brokerAddr);
+ }
+ };
+
+ if (messageStore != null) {
+ messageStore.setSendMessageBackHook(sendMessageBackHook);
+ }
+ }
+
+ private void initialTransaction() {
+ this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class);
+ if (null == this.transactionalMessageService) {
+ this.transactionalMessageService = new TransactionalMessageServiceImpl(
+ new TransactionalMessageBridge(this, this.getMessageStore()));
+ LOG.warn("Load default transaction message hook service: {}",
+ TransactionalMessageServiceImpl.class.getSimpleName());
+ }
+ this.transactionalMessageCheckListener = ServiceProvider.loadClass(
+ AbstractTransactionalMessageCheckListener.class);
+ if (null == this.transactionalMessageCheckListener) {
+ this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener();
+ LOG.warn("Load default discard message hook service: {}",
+ DefaultTransactionalMessageCheckListener.class.getSimpleName());
+ }
+ this.transactionalMessageCheckListener.setBrokerController(this);
+ this.transactionalMessageCheckService = new TransactionalMessageCheckService(this);
+ this.transactionMetricsFlushService = new TransactionMetricsFlushService(this);
+ this.transactionMetricsFlushService.start();
+
+ }
+
+ private void initialAcl() {
+ if (!this.brokerConfig.isAclEnable()) {
+ LOG.info("The broker dose not enable acl");
+ return;
+ }
+
+ List accessValidators = ServiceProvider.load(AccessValidator.class);
+ if (accessValidators.isEmpty()) {
+ LOG.info("ServiceProvider loaded no AccessValidator, using default org.apache.rocketmq.acl.plain.PlainAccessValidator");
+ accessValidators.add(new PlainAccessValidator());
+ }
+
+ for (AccessValidator accessValidator : accessValidators) {
+ final AccessValidator validator = accessValidator;
+ accessValidatorMap.put(validator.getClass(), validator);
+ this.registerServerRPCHook(new RPCHook() {
+
+ @Override
+ public void doBeforeRequest(String remoteAddr, RemotingCommand request) {
+ //Do not catch the exception
+ validator.validate(validator.parse(request, remoteAddr));
+ }
+
+ @Override
+ public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {
+ }
+
+ });
+ }
+ }
+
+ private void initialRpcHooks() {
+
+ List rpcHooks = ServiceProvider.load(RPCHook.class);
+ if (rpcHooks == null || rpcHooks.isEmpty()) {
+ return;
+ }
+ for (RPCHook rpcHook : rpcHooks) {
+ this.registerServerRPCHook(rpcHook);
+ }
+ }
+
+ public void registerProcessor() {
+ /*
+ * SendMessageProcessor
+ */
+ sendMessageProcessor.registerSendMessageHook(sendMessageHookList);
+ sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
+
+ this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
+ /**
+ * PullMessageProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, this.pullMessageProcessor, this.litePullMessageExecutor);
+ this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
+ /**
+ * PeekMessageProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.PEEK_MESSAGE, this.peekMessageProcessor, this.pullMessageExecutor);
+ /**
+ * PopMessageProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.POP_MESSAGE, this.popMessageProcessor, this.pullMessageExecutor);
+
+ /**
+ * AckMessageProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+
+ this.remotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+ /**
+ * ChangeInvisibleTimeProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
+ /**
+ * notificationProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.NOTIFICATION, this.notificationProcessor, this.pullMessageExecutor);
+
+ /**
+ * pollingInfoProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.POLLING_INFO, this.pollingInfoProcessor, this.pullMessageExecutor);
+
+ /**
+ * ReplyMessageProcessor
+ */
+
+ replyMessageProcessor.registerSendMessageHook(sendMessageHookList);
+
+ this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
+
+ /**
+ * QueryMessageProcessor
+ */
+ NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this);
+ this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
+
+ this.fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
+
+ /**
+ * ClientManageProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
+ this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
+
+ this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
+
+ /**
+ * ConsumerManageProcessor
+ */
+ ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this);
+ this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+
+ this.fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+
+ /**
+ * QueryAssignmentProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
+ this.remotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
+
+ /**
+ * EndTransactionProcessor
+ */
+ this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
+
+ /*
+ * Default
+ */
+ AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this);
+ this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+ this.fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+ }
+
+ public BrokerStats getBrokerStats() {
+ return brokerStats;
+ }
+
+ public void setBrokerStats(BrokerStats brokerStats) {
+ this.brokerStats = brokerStats;
+ }
+
+ public void protectBroker() {
+ if (this.brokerConfig.isDisableConsumeIfConsumerReadSlowly()) {
+ for (Map.Entry next : this.brokerStatsManager.getMomentStatsItemSetFallSize().getStatsItemTable().entrySet()) {
+ final long fallBehindBytes = next.getValue().getValue().get();
+ if (fallBehindBytes > this.brokerConfig.getConsumerFallbehindThreshold()) {
+ final String[] split = next.getValue().getStatsKey().split("@");
+ final String group = split[2];
+ LOG_PROTECTION.info("[PROTECT_BROKER] the consumer[{}] consume slowly, {} bytes, disable it", group, fallBehindBytes);
+ this.subscriptionGroupManager.disableConsume(group);
+ }
+ }
+ }
+ }
+
+ public long headSlowTimeMills(BlockingQueue q) {
+ long slowTimeMills = 0;
+ final Runnable peek = q.peek();
+ if (peek != null) {
+ RequestTask rt = BrokerFastFailure.castRunnable(peek);
+ slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp();
+ }
+
+ if (slowTimeMills < 0) {
+ slowTimeMills = 0;
+ }
+
+ return slowTimeMills;
+ }
+
+ public long headSlowTimeMills4SendThreadPoolQueue() {
+ return this.headSlowTimeMills(this.sendThreadPoolQueue);
+ }
+
+ public long headSlowTimeMills4PullThreadPoolQueue() {
+ return this.headSlowTimeMills(this.pullThreadPoolQueue);
+ }
+
+ public long headSlowTimeMills4LitePullThreadPoolQueue() {
+ return this.headSlowTimeMills(this.litePullThreadPoolQueue);
+ }
+
+ public long headSlowTimeMills4QueryThreadPoolQueue() {
+ return this.headSlowTimeMills(this.queryThreadPoolQueue);
+ }
+
+ public void printWaterMark() {
+ LOG_WATER_MARK.info("[WATERMARK] Send Queue Size: {} SlowTimeMills: {}", this.sendThreadPoolQueue.size(), headSlowTimeMills4SendThreadPoolQueue());
+ LOG_WATER_MARK.info("[WATERMARK] Pull Queue Size: {} SlowTimeMills: {}", this.pullThreadPoolQueue.size(), headSlowTimeMills4PullThreadPoolQueue());
+ LOG_WATER_MARK.info("[WATERMARK] Query Queue Size: {} SlowTimeMills: {}", this.queryThreadPoolQueue.size(), headSlowTimeMills4QueryThreadPoolQueue());
+ LOG_WATER_MARK.info("[WATERMARK] Lite Pull Queue Size: {} SlowTimeMills: {}", this.litePullThreadPoolQueue.size(), headSlowTimeMills4LitePullThreadPoolQueue());
+ LOG_WATER_MARK.info("[WATERMARK] Transaction Queue Size: {} SlowTimeMills: {}", this.endTransactionThreadPoolQueue.size(), headSlowTimeMills(this.endTransactionThreadPoolQueue));
+ LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue));
+ LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue));
+ LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue));
+ LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue));
+ }
+
+ public MessageStore getMessageStore() {
+ return messageStore;
+ }
+
+ public void setMessageStore(MessageStore messageStore) {
+ this.messageStore = messageStore;
+ }
+
+ protected void printMasterAndSlaveDiff() {
+ if (messageStore.getHaService() != null && messageStore.getHaService().getConnectionCount().get() > 0) {
+ long diff = this.messageStore.slaveFallBehindMuch();
+ LOG.info("CommitLog: slave fall behind master {}bytes", diff);
+ }
}
public Broker2Client getBroker2Client() {
@@ -545,10 +1223,22 @@ public ConsumerFilterManager getConsumerFilterManager() {
return consumerFilterManager;
}
+ public ConsumerOrderInfoManager getConsumerOrderInfoManager() {
+ return consumerOrderInfoManager;
+ }
+
+ public PopInflightMessageCounter getPopInflightMessageCounter() {
+ return popInflightMessageCounter;
+ }
+
public ConsumerOffsetManager getConsumerOffsetManager() {
return consumerOffsetManager;
}
+ public BroadcastOffsetManager getBroadcastOffsetManager() {
+ return broadcastOffsetManager;
+ }
+
public MessageStoreConfig getMessageStoreConfig() {
return messageStoreConfig;
}
@@ -561,185 +1251,897 @@ public void setFastRemotingServer(RemotingServer fastRemotingServer) {
this.fastRemotingServer = fastRemotingServer;
}
+ public RemotingServer getFastRemotingServer() {
+ return fastRemotingServer;
+ }
+
public PullMessageProcessor getPullMessageProcessor() {
return pullMessageProcessor;
}
- public PullRequestHoldService getPullRequestHoldService() {
- return pullRequestHoldService;
+ public PullRequestHoldService getPullRequestHoldService() {
+ return pullRequestHoldService;
+ }
+
+ public void setSubscriptionGroupManager(SubscriptionGroupManager subscriptionGroupManager) {
+ this.subscriptionGroupManager = subscriptionGroupManager;
+ }
+
+ public SubscriptionGroupManager getSubscriptionGroupManager() {
+ return subscriptionGroupManager;
+ }
+
+ public PopMessageProcessor getPopMessageProcessor() {
+ return popMessageProcessor;
+ }
+
+ public NotificationProcessor getNotificationProcessor() {
+ return notificationProcessor;
+ }
+
+ public TimerMessageStore getTimerMessageStore() {
+ return timerMessageStore;
+ }
+
+ public void setTimerMessageStore(TimerMessageStore timerMessageStore) {
+ this.timerMessageStore = timerMessageStore;
+ }
+
+ public AckMessageProcessor getAckMessageProcessor() {
+ return ackMessageProcessor;
+ }
+
+ public ChangeInvisibleTimeProcessor getChangeInvisibleTimeProcessor() {
+ return changeInvisibleTimeProcessor;
+ }
+
+ protected void shutdownBasicService() {
+
+ shutdown = true;
+
+ this.unregisterBrokerAll();
+
+ if (this.shutdownHook != null) {
+ this.shutdownHook.beforeShutdown(this);
+ }
+
+ if (this.remotingServer != null) {
+ this.remotingServer.shutdown();
+ }
+
+ if (this.fastRemotingServer != null) {
+ this.fastRemotingServer.shutdown();
+ }
+
+ if (this.brokerMetricsManager != null) {
+ this.brokerMetricsManager.shutdown();
+ }
+
+ if (this.brokerStatsManager != null) {
+ this.brokerStatsManager.shutdown();
+ }
+
+ if (this.clientHousekeepingService != null) {
+ this.clientHousekeepingService.shutdown();
+ }
+
+ if (this.pullRequestHoldService != null) {
+ this.pullRequestHoldService.shutdown();
+ }
+
+ {
+ this.popMessageProcessor.getPopLongPollingService().shutdown();
+ this.popMessageProcessor.getQueueLockManager().shutdown();
+ }
+
+ {
+ this.popMessageProcessor.getPopBufferMergeService().shutdown();
+ this.ackMessageProcessor.shutdownPopReviveService();
+ }
+
+ if (this.transactionalMessageService != null) {
+ this.transactionalMessageService.close();
+ }
+
+ if (this.notificationProcessor != null) {
+ this.notificationProcessor.getPopLongPollingService().shutdown();
+ }
+
+ if (this.consumerIdsChangeListener != null) {
+ this.consumerIdsChangeListener.shutdown();
+ }
+
+ if (this.topicQueueMappingCleanService != null) {
+ this.topicQueueMappingCleanService.shutdown();
+ }
+ //it is better to make sure the timerMessageStore shutdown firstly
+ if (this.timerMessageStore != null) {
+ this.timerMessageStore.shutdown();
+ }
+ if (this.fileWatchService != null) {
+ this.fileWatchService.shutdown();
+ }
+
+ if (this.broadcastOffsetManager != null) {
+ this.broadcastOffsetManager.shutdown();
+ }
+
+ if (this.messageStore != null) {
+ this.messageStore.shutdown();
+ }
+
+ if (this.replicasManager != null) {
+ this.replicasManager.shutdown();
+ }
+
+ shutdownScheduledExecutorService(this.scheduledExecutorService);
+
+ if (this.sendMessageExecutor != null) {
+ this.sendMessageExecutor.shutdown();
+ }
+
+ if (this.litePullMessageExecutor != null) {
+ this.litePullMessageExecutor.shutdown();
+ }
+
+ if (this.pullMessageExecutor != null) {
+ this.pullMessageExecutor.shutdown();
+ }
+
+ if (this.replyMessageExecutor != null) {
+ this.replyMessageExecutor.shutdown();
+ }
+
+ if (this.putMessageFutureExecutor != null) {
+ this.putMessageFutureExecutor.shutdown();
+ }
+
+ if (this.ackMessageExecutor != null) {
+ this.ackMessageExecutor.shutdown();
+ }
+
+ if (this.adminBrokerExecutor != null) {
+ this.adminBrokerExecutor.shutdown();
+ }
+
+ if (this.brokerFastFailure != null) {
+ this.brokerFastFailure.shutdown();
+ }
+
+ if (this.consumerFilterManager != null) {
+ this.consumerFilterManager.persist();
+ }
+
+ if (this.consumerOrderInfoManager != null) {
+ this.consumerOrderInfoManager.persist();
+ }
+
+ if (this.scheduleMessageService != null) {
+ this.scheduleMessageService.persist();
+ this.scheduleMessageService.shutdown();
+ }
+
+ if (this.clientManageExecutor != null) {
+ this.clientManageExecutor.shutdown();
+ }
+
+ if (this.queryMessageExecutor != null) {
+ this.queryMessageExecutor.shutdown();
+ }
+
+ if (this.heartbeatExecutor != null) {
+ this.heartbeatExecutor.shutdown();
+ }
+
+ if (this.consumerManageExecutor != null) {
+ this.consumerManageExecutor.shutdown();
+ }
+
+ if (this.transactionalMessageCheckService != null) {
+ this.transactionalMessageCheckService.shutdown(false);
+ }
+
+ if (this.endTransactionExecutor != null) {
+ this.endTransactionExecutor.shutdown();
+ }
+
+ if (this.transactionMetricsFlushService != null) {
+ this.transactionMetricsFlushService.shutdown();
+ }
+
+ if (this.escapeBridge != null) {
+ escapeBridge.shutdown();
+ }
+
+ if (this.topicRouteInfoManager != null) {
+ this.topicRouteInfoManager.shutdown();
+ }
+
+ if (this.brokerPreOnlineService != null && !this.brokerPreOnlineService.isStopped()) {
+ this.brokerPreOnlineService.shutdown();
+ }
+
+ if (this.coldDataPullRequestHoldService != null) {
+ this.coldDataPullRequestHoldService.shutdown();
+ }
+
+ if (this.coldDataCgCtrService != null) {
+ this.coldDataCgCtrService.shutdown();
+ }
+
+ shutdownScheduledExecutorService(this.syncBrokerMemberGroupExecutorService);
+ shutdownScheduledExecutorService(this.brokerHeartbeatExecutorService);
+
+ if (this.topicConfigManager != null) {
+ this.topicConfigManager.persist();
+ this.topicConfigManager.stop();
+ }
+
+ if (this.subscriptionGroupManager != null) {
+ this.subscriptionGroupManager.persist();
+ this.subscriptionGroupManager.stop();
+ }
+
+ if (this.consumerOffsetManager != null) {
+ this.consumerOffsetManager.persist();
+ this.consumerOffsetManager.stop();
+ }
+
+ for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
+ if (brokerAttachedPlugin != null) {
+ brokerAttachedPlugin.shutdown();
+ }
+ }
+ }
+
+ public void shutdown() {
+
+ shutdownBasicService();
+
+ for (ScheduledFuture scheduledFuture : scheduledFutures) {
+ scheduledFuture.cancel(true);
+ }
+
+ if (this.brokerOuterAPI != null) {
+ this.brokerOuterAPI.shutdown();
+ }
+ }
+
+ protected void shutdownScheduledExecutorService(ScheduledExecutorService scheduledExecutorService) {
+ if (scheduledExecutorService == null) {
+ return;
+ }
+ scheduledExecutorService.shutdown();
+ try {
+ scheduledExecutorService.awaitTermination(5000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignore) {
+ BrokerController.LOG.warn("shutdown ScheduledExecutorService was Interrupted! ", ignore);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ protected void unregisterBrokerAll() {
+ this.brokerOuterAPI.unregisterBrokerAll(
+ this.brokerConfig.getBrokerClusterName(),
+ this.getBrokerAddr(),
+ this.brokerConfig.getBrokerName(),
+ this.brokerConfig.getBrokerId());
+ }
+
+ public String getBrokerAddr() {
+ return this.brokerConfig.getBrokerIP1() + ":" + this.nettyServerConfig.getListenPort();
+ }
+
+ protected void startBasicService() throws Exception {
+
+ if (this.messageStore != null) {
+ this.messageStore.start();
+ }
+
+ if (this.timerMessageStore != null) {
+ this.timerMessageStore.start();
+ }
+
+ if (this.replicasManager != null) {
+ this.replicasManager.start();
+ }
+
+ if (remotingServerStartLatch != null) {
+ remotingServerStartLatch.await();
+ }
+
+ if (this.remotingServer != null) {
+ this.remotingServer.start();
+
+ // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config
+ if (null != nettyServerConfig && 0 == nettyServerConfig.getListenPort()) {
+ nettyServerConfig.setListenPort(remotingServer.localListenPort());
+ }
+ }
+
+ if (this.fastRemotingServer != null) {
+ this.fastRemotingServer.start();
+ }
+
+ this.storeHost = new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort());
+
+ for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
+ if (brokerAttachedPlugin != null) {
+ brokerAttachedPlugin.start();
+ }
+ }
+
+ if (this.popMessageProcessor != null) {
+ this.popMessageProcessor.getPopLongPollingService().start();
+ this.popMessageProcessor.getPopBufferMergeService().start();
+ this.popMessageProcessor.getQueueLockManager().start();
+ }
+
+ if (this.ackMessageProcessor != null) {
+ this.ackMessageProcessor.startPopReviveService();
+ }
+
+ if (this.notificationProcessor != null) {
+ this.notificationProcessor.getPopLongPollingService().start();
+ }
+
+ if (this.topicQueueMappingCleanService != null) {
+ this.topicQueueMappingCleanService.start();
+ }
+
+ if (this.fileWatchService != null) {
+ this.fileWatchService.start();
+ }
+
+ if (this.pullRequestHoldService != null) {
+ this.pullRequestHoldService.start();
+ }
+
+ if (this.clientHousekeepingService != null) {
+ this.clientHousekeepingService.start();
+ }
+
+ if (this.brokerStatsManager != null) {
+ this.brokerStatsManager.start();
+ }
+
+ if (this.brokerFastFailure != null) {
+ this.brokerFastFailure.start();
+ }
+
+ if (this.broadcastOffsetManager != null) {
+ this.broadcastOffsetManager.start();
+ }
+
+ if (this.escapeBridge != null) {
+ this.escapeBridge.start();
+ }
+
+ if (this.topicRouteInfoManager != null) {
+ this.topicRouteInfoManager.start();
+ }
+
+ if (this.brokerPreOnlineService != null) {
+ this.brokerPreOnlineService.start();
+ }
+
+ if (this.coldDataPullRequestHoldService != null) {
+ this.coldDataPullRequestHoldService.start();
+ }
+
+ if (this.coldDataCgCtrService != null) {
+ this.coldDataCgCtrService.start();
+ }
+ }
+
+ public void start() throws Exception {
+
+ this.shouldStartTime = System.currentTimeMillis() + messageStoreConfig.getDisappearTimeAfterStart();
+
+ if (messageStoreConfig.getTotalReplicas() > 1 && this.brokerConfig.isEnableSlaveActingMaster()) {
+ isIsolated = true;
+ }
+
+ if (this.brokerOuterAPI != null) {
+ this.brokerOuterAPI.start();
+ }
+
+ startBasicService();
+
+ if (!isIsolated && !this.messageStoreConfig.isEnableDLegerCommitLog() && !this.messageStoreConfig.isDuplicationEnable()) {
+ changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == MixAll.MASTER_ID);
+ this.registerBrokerAll(true, false, true);
+ }
+
+ scheduledFutures.add(this.scheduledExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(this.getBrokerIdentity()) {
+ @Override
+ public void run0() {
+ try {
+ if (System.currentTimeMillis() < shouldStartTime) {
+ BrokerController.LOG.info("Register to namesrv after {}", shouldStartTime);
+ return;
+ }
+ if (isIsolated) {
+ BrokerController.LOG.info("Skip register for broker is isolated");
+ return;
+ }
+ BrokerController.this.registerBrokerAll(true, false, brokerConfig.isForceRegister());
+ } catch (Throwable e) {
+ BrokerController.LOG.error("registerBrokerAll Exception", e);
+ }
+ }
+ }, 1000 * 10, Math.max(10000, Math.min(brokerConfig.getRegisterNameServerPeriod(), 60000)), TimeUnit.MILLISECONDS));
+
+ if (this.brokerConfig.isEnableSlaveActingMaster()) {
+ scheduleSendHeartbeat();
+
+ scheduledFutures.add(this.syncBrokerMemberGroupExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(this.getBrokerIdentity()) {
+ @Override
+ public void run0() {
+ try {
+ BrokerController.this.syncBrokerMemberGroup();
+ } catch (Throwable e) {
+ BrokerController.LOG.error("sync BrokerMemberGroup error. ", e);
+ }
+ }
+ }, 1000, this.brokerConfig.getSyncBrokerMemberGroupPeriod(), TimeUnit.MILLISECONDS));
+ }
+
+ if (this.brokerConfig.isEnableControllerMode()) {
+ scheduleSendHeartbeat();
+ }
+
+ if (brokerConfig.isSkipPreOnline()) {
+ startServiceWithoutCondition();
+ }
+
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ BrokerController.this.brokerOuterAPI.refreshMetadata();
+ } catch (Exception e) {
+ LOG.error("ScheduledTask refresh metadata exception", e);
+ }
+ }
+ }, 10, 5, TimeUnit.SECONDS);
+ }
+
+ protected void scheduleSendHeartbeat() {
+ scheduledFutures.add(this.brokerHeartbeatExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(this.getBrokerIdentity()) {
+ @Override
+ public void run0() {
+ if (isIsolated) {
+ return;
+ }
+ try {
+ BrokerController.this.sendHeartbeat();
+ } catch (Exception e) {
+ BrokerController.LOG.error("sendHeartbeat Exception", e);
+ }
+
+ }
+ }, 1000, brokerConfig.getBrokerHeartbeatInterval(), TimeUnit.MILLISECONDS));
+ }
+
+ public synchronized void registerSingleTopicAll(final TopicConfig topicConfig) {
+ TopicConfig tmpTopic = topicConfig;
+ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
+ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
+ // Copy the topic config and modify the perm
+ tmpTopic = new TopicConfig(topicConfig);
+ tmpTopic.setPerm(topicConfig.getPerm() & this.brokerConfig.getBrokerPermission());
+ }
+ this.brokerOuterAPI.registerSingleTopicAll(this.brokerConfig.getBrokerName(), tmpTopic, 3000);
+ }
+
+ public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) {
+ this.registerIncrementBrokerData(Collections.singletonList(topicConfig), dataVersion);
}
- public SubscriptionGroupManager getSubscriptionGroupManager() {
- return subscriptionGroupManager;
- }
+ public synchronized void registerIncrementBrokerData(List topicConfigList, DataVersion dataVersion) {
+ if (topicConfigList == null || topicConfigList.isEmpty()) {
+ return;
+ }
- public void shutdown() {
- if (this.brokerStatsManager != null) {
- this.brokerStatsManager.shutdown();
+ TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();
+ topicConfigSerializeWrapper.setDataVersion(dataVersion);
+
+ ConcurrentMap topicConfigTable = topicConfigList.stream()
+ .map(topicConfig -> {
+ TopicConfig registerTopicConfig;
+ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
+ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
+ registerTopicConfig =
+ new TopicConfig(topicConfig.getTopicName(),
+ topicConfig.getReadQueueNums(),
+ topicConfig.getWriteQueueNums(),
+ topicConfig.getPerm()
+ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
+ } else {
+ registerTopicConfig = new TopicConfig(topicConfig);
+ }
+ return registerTopicConfig;
+ })
+ .collect(Collectors.toConcurrentMap(TopicConfig::getTopicName, Function.identity()));
+ topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);
+
+ Map topicQueueMappingInfoMap = topicConfigList.stream()
+ .map(TopicConfig::getTopicName)
+ .map(topicName -> Optional.ofNullable(this.topicQueueMappingManager.getTopicQueueMapping(topicName))
+ .map(info -> new AbstractMap.SimpleImmutableEntry<>(topicName, TopicQueueMappingDetail.cloneAsMappingInfo(info)))
+ .orElse(null))
+ .filter(Objects::nonNull)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ if (!topicQueueMappingInfoMap.isEmpty()) {
+ topicConfigSerializeWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);
}
- if (this.clientHousekeepingService != null) {
- this.clientHousekeepingService.shutdown();
+ doRegisterBrokerAll(true, false, topicConfigSerializeWrapper);
+ }
+
+ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) {
+ ConcurrentMap topicConfigMap = this.getTopicConfigManager().getTopicConfigTable();
+ ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>();
+
+ for (TopicConfig topicConfig : topicConfigMap.values()) {
+ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
+ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
+ topicConfigTable.put(topicConfig.getTopicName(),
+ new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
+ topicConfig.getPerm() & getBrokerConfig().getBrokerPermission()));
+ } else {
+ topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+ }
+
+ if (this.brokerConfig.isEnableSplitRegistration()
+ && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) {
+ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable);
+ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);
+ topicConfigTable.clear();
+ }
}
- if (this.pullRequestHoldService != null) {
- this.pullRequestHoldService.shutdown();
+ Map topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream()
+ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().
+ buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap);
+ if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
+ this.getBrokerAddr(),
+ this.brokerConfig.getBrokerName(),
+ this.brokerConfig.getBrokerId(),
+ this.brokerConfig.getRegisterBrokerTimeoutMills(),
+ this.brokerConfig.isInBrokerContainer())) {
+ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);
}
+ }
- if (this.remotingServer != null) {
- this.remotingServer.shutdown();
+ protected void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway,
+ TopicConfigSerializeWrapper topicConfigWrapper) {
+
+ if (shutdown) {
+ BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more.");
+ return;
}
+ List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll(
+ this.brokerConfig.getBrokerClusterName(),
+ this.getBrokerAddr(),
+ this.brokerConfig.getBrokerName(),
+ this.brokerConfig.getBrokerId(),
+ this.getHAServerAddr(),
+ topicConfigWrapper,
+ Lists.newArrayList(),
+ oneway,
+ this.brokerConfig.getRegisterBrokerTimeoutMills(),
+ this.brokerConfig.isEnableSlaveActingMaster(),
+ this.brokerConfig.isCompressedRegister(),
+ this.brokerConfig.isEnableSlaveActingMaster() ? this.brokerConfig.getBrokerNotActiveTimeoutMillis() : null,
+ this.getBrokerIdentity());
- if (this.fastRemotingServer != null) {
- this.fastRemotingServer.shutdown();
+ handleRegisterBrokerResult(registerBrokerResultList, checkOrderConfig);
+ }
+
+ protected void sendHeartbeat() {
+ if (this.brokerConfig.isEnableControllerMode()) {
+ this.replicasManager.sendHeartbeatToController();
}
- if (this.messageStore != null) {
- this.messageStore.shutdown();
+ if (this.brokerConfig.isEnableSlaveActingMaster()) {
+ if (this.brokerConfig.isCompatibleWithOldNameSrv()) {
+ this.brokerOuterAPI.sendHeartbeatViaDataVersion(
+ this.brokerConfig.getBrokerClusterName(),
+ this.getBrokerAddr(),
+ this.brokerConfig.getBrokerName(),
+ this.brokerConfig.getBrokerId(),
+ this.brokerConfig.getSendHeartbeatTimeoutMillis(),
+ this.getTopicConfigManager().getDataVersion(),
+ this.brokerConfig.isInBrokerContainer());
+ } else {
+ this.brokerOuterAPI.sendHeartbeat(
+ this.brokerConfig.getBrokerClusterName(),
+ this.getBrokerAddr(),
+ this.brokerConfig.getBrokerName(),
+ this.brokerConfig.getBrokerId(),
+ this.brokerConfig.getSendHeartbeatTimeoutMillis(),
+ this.brokerConfig.isInBrokerContainer());
+ }
}
+ }
- this.scheduledExecutorService.shutdown();
+ protected void syncBrokerMemberGroup() {
try {
- this.scheduledExecutorService.awaitTermination(5000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
+ brokerMemberGroup = this.getBrokerOuterAPI()
+ .syncBrokerMemberGroup(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName(), this.brokerConfig.isCompatibleWithOldNameSrv());
+ } catch (Exception e) {
+ BrokerController.LOG.error("syncBrokerMemberGroup from namesrv failed, ", e);
+ return;
}
-
- this.unregisterBrokerAll();
-
- if (this.sendMessageExecutor != null) {
- this.sendMessageExecutor.shutdown();
+ if (brokerMemberGroup == null || brokerMemberGroup.getBrokerAddrs().size() == 0) {
+ BrokerController.LOG.warn("Couldn't find any broker member from namesrv in {}/{}", this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName());
+ return;
}
+ this.messageStore.setAliveReplicaNumInGroup(calcAliveBrokerNumInGroup(brokerMemberGroup.getBrokerAddrs()));
- if (this.pullMessageExecutor != null) {
- this.pullMessageExecutor.shutdown();
+ if (!this.isIsolated) {
+ long minBrokerId = brokerMemberGroup.minimumBrokerId();
+ this.updateMinBroker(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
}
+ }
- if (this.adminBrokerExecutor != null) {
- this.adminBrokerExecutor.shutdown();
+ private int calcAliveBrokerNumInGroup(Map brokerAddrTable) {
+ if (brokerAddrTable.containsKey(this.brokerConfig.getBrokerId())) {
+ return brokerAddrTable.size();
+ } else {
+ return brokerAddrTable.size() + 1;
}
+ }
- if (this.brokerOuterAPI != null) {
- this.brokerOuterAPI.shutdown();
+ protected void handleRegisterBrokerResult(List registerBrokerResultList,
+ boolean checkOrderConfig) {
+ for (RegisterBrokerResult registerBrokerResult : registerBrokerResultList) {
+ if (registerBrokerResult != null) {
+ if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) {
+ this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr());
+ this.messageStore.updateMasterAddress(registerBrokerResult.getMasterAddr());
+ }
+
+ this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr());
+ if (checkOrderConfig) {
+ this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable());
+ }
+ break;
+ }
}
+ }
- this.consumerOffsetManager.persist();
+ private boolean needRegister(final String clusterName,
+ final String brokerAddr,
+ final String brokerName,
+ final long brokerId,
+ final int timeoutMills,
+ final boolean isInBrokerContainer) {
- if (this.filterServerManager != null) {
- this.filterServerManager.shutdown();
+ TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();
+ List changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills, isInBrokerContainer);
+ boolean needRegister = false;
+ for (Boolean changed : changeList) {
+ if (changed) {
+ needRegister = true;
+ break;
+ }
}
+ return needRegister;
+ }
- if (this.brokerFastFailure != null) {
- this.brokerFastFailure.shutdown();
- }
+ public void startService(long minBrokerId, String minBrokerAddr) {
+ BrokerController.LOG.info("{} start service, min broker id is {}, min broker addr: {}",
+ this.brokerConfig.getCanonicalName(), minBrokerId, minBrokerAddr);
+ this.minBrokerIdInGroup = minBrokerId;
+ this.minBrokerAddrInGroup = minBrokerAddr;
- if (this.consumerFilterManager != null) {
- this.consumerFilterManager.persist();
- }
+ this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == minBrokerId);
+ this.registerBrokerAll(true, false, brokerConfig.isForceRegister());
+
+ isIsolated = false;
}
- private void unregisterBrokerAll() {
- this.brokerOuterAPI.unregisterBrokerAll(
- this.brokerConfig.getBrokerClusterName(),
- this.getBrokerAddr(),
- this.brokerConfig.getBrokerName(),
- this.brokerConfig.getBrokerId());
+ public void startServiceWithoutCondition() {
+ BrokerController.LOG.info("{} start service", this.brokerConfig.getCanonicalName());
+
+ this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == MixAll.MASTER_ID);
+ this.registerBrokerAll(true, false, brokerConfig.isForceRegister());
+
+ isIsolated = false;
}
- public String getBrokerAddr() {
- return this.brokerConfig.getBrokerIP1() + ":" + this.nettyServerConfig.getListenPort();
+ public void stopService() {
+ BrokerController.LOG.info("{} stop service", this.getBrokerConfig().getCanonicalName());
+ isIsolated = true;
+ this.changeSpecialServiceStatus(false);
}
- public void start() throws Exception {
- if (this.messageStore != null) {
- this.messageStore.start();
+ public boolean isSpecialServiceRunning() {
+ if (isScheduleServiceStart() && isTransactionCheckServiceStart()) {
+ return true;
}
- if (this.remotingServer != null) {
- this.remotingServer.start();
- }
+ return this.ackMessageProcessor != null && this.ackMessageProcessor.isPopReviveServiceRunning();
+ }
- if (this.fastRemotingServer != null) {
- this.fastRemotingServer.start();
+ private void onMasterOffline() {
+ // close channels with master broker
+ String masterAddr = this.slaveSynchronize.getMasterAddr();
+ if (masterAddr != null) {
+ this.brokerOuterAPI.getRemotingClient().closeChannels(
+ Arrays.asList(masterAddr, MixAll.brokerVIPChannel(true, masterAddr)));
}
+ // master not available, stop sync
+ this.slaveSynchronize.setMasterAddr(null);
+ this.messageStore.updateHaMasterAddress(null);
+ }
- if (this.brokerOuterAPI != null) {
- this.brokerOuterAPI.start();
- }
+ private void onMasterOnline(String masterAddr, String masterHaAddr) {
+ boolean needSyncMasterFlushOffset = this.messageStore.getMasterFlushedOffset() == 0
+ && this.messageStoreConfig.isSyncMasterFlushOffsetWhenStartup();
+ if (masterHaAddr == null || needSyncMasterFlushOffset) {
+ try {
+ BrokerSyncInfo brokerSyncInfo = this.brokerOuterAPI.retrieveBrokerHaInfo(masterAddr);
- if (this.pullRequestHoldService != null) {
- this.pullRequestHoldService.start();
+ if (needSyncMasterFlushOffset) {
+ LOG.info("Set master flush offset in slave to {}", brokerSyncInfo.getMasterFlushOffset());
+ this.messageStore.setMasterFlushedOffset(brokerSyncInfo.getMasterFlushOffset());
+ }
+
+ if (masterHaAddr == null) {
+ this.messageStore.updateHaMasterAddress(brokerSyncInfo.getMasterHaAddress());
+ this.messageStore.updateMasterAddress(brokerSyncInfo.getMasterAddress());
+ }
+ } catch (Exception e) {
+ LOG.error("retrieve master ha info exception, {}", e);
+ }
}
- if (this.clientHousekeepingService != null) {
- this.clientHousekeepingService.start();
+ // set master HA address.
+ if (masterHaAddr != null) {
+ this.messageStore.updateHaMasterAddress(masterHaAddr);
}
- if (this.filterServerManager != null) {
- this.filterServerManager.start();
+ // wakeup HAClient
+ this.messageStore.wakeupHAClient();
+ }
+
+ private void onMinBrokerChange(long minBrokerId, String minBrokerAddr, String offlineBrokerAddr,
+ String masterHaAddr) {
+ LOG.info("Min broker changed, old: {}-{}, new {}-{}",
+ this.minBrokerIdInGroup, this.minBrokerAddrInGroup, minBrokerId, minBrokerAddr);
+
+ this.minBrokerIdInGroup = minBrokerId;
+ this.minBrokerAddrInGroup = minBrokerAddr;
+
+ this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == this.minBrokerIdInGroup);
+
+ if (offlineBrokerAddr != null && offlineBrokerAddr.equals(this.slaveSynchronize.getMasterAddr())) {
+ // master offline
+ onMasterOffline();
}
- this.registerBrokerAll(true, false);
+ if (minBrokerId == MixAll.MASTER_ID && minBrokerAddr != null) {
+ // master online
+ onMasterOnline(minBrokerAddr, masterHaAddr);
+ }
- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ // notify PullRequest on hold to pull from master.
+ if (this.minBrokerIdInGroup == MixAll.MASTER_ID) {
+ this.pullRequestHoldService.notifyMasterOnline();
+ }
+ }
- @Override
- public void run() {
+ public void updateMinBroker(long minBrokerId, String minBrokerAddr) {
+ if (brokerConfig.isEnableSlaveActingMaster() && brokerConfig.getBrokerId() != MixAll.MASTER_ID) {
+ if (lock.tryLock()) {
try {
- BrokerController.this.registerBrokerAll(true, false);
- } catch (Throwable e) {
- log.error("registerBrokerAll Exception", e);
+ if (minBrokerId != this.minBrokerIdInGroup) {
+ String offlineBrokerAddr = null;
+ if (minBrokerId > this.minBrokerIdInGroup) {
+ offlineBrokerAddr = this.minBrokerAddrInGroup;
+ }
+ onMinBrokerChange(minBrokerId, minBrokerAddr, offlineBrokerAddr, null);
+ }
+ } finally {
+ lock.unlock();
}
}
- }, 1000 * 10, 1000 * 30, TimeUnit.MILLISECONDS);
-
- if (this.brokerStatsManager != null) {
- this.brokerStatsManager.start();
}
+ }
- if (this.brokerFastFailure != null) {
- this.brokerFastFailure.start();
+ public void updateMinBroker(long minBrokerId, String minBrokerAddr, String offlineBrokerAddr,
+ String masterHaAddr) {
+ if (brokerConfig.isEnableSlaveActingMaster() && brokerConfig.getBrokerId() != MixAll.MASTER_ID) {
+ try {
+ if (lock.tryLock(3000, TimeUnit.MILLISECONDS)) {
+ try {
+ if (minBrokerId != this.minBrokerIdInGroup) {
+ onMinBrokerChange(minBrokerId, minBrokerAddr, offlineBrokerAddr, masterHaAddr);
+ }
+ } finally {
+ lock.unlock();
+ }
+
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Update min broker error, {}", e);
+ }
}
}
- public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway) {
- TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();
+ public void changeSpecialServiceStatus(boolean shouldStart) {
- if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
- || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
- ConcurrentHashMap topicConfigTable = new ConcurrentHashMap();
- for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) {
- TopicConfig tmp =
- new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
- this.brokerConfig.getBrokerPermission());
- topicConfigTable.put(topicConfig.getTopicName(), tmp);
+ for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
+ if (brokerAttachedPlugin != null) {
+ brokerAttachedPlugin.statusChanged(shouldStart);
}
- topicConfigWrapper.setTopicConfigTable(topicConfigTable);
}
- RegisterBrokerResult registerBrokerResult = this.brokerOuterAPI.registerBrokerAll(
- this.brokerConfig.getBrokerClusterName(),
- this.getBrokerAddr(),
- this.brokerConfig.getBrokerName(),
- this.brokerConfig.getBrokerId(),
- this.getHAServerAddr(),
- topicConfigWrapper,
- this.filterServerManager.buildNewFilterServerList(),
- oneway,
- this.brokerConfig.getRegisterBrokerTimeoutMills());
+ changeScheduleServiceStatus(shouldStart);
- if (registerBrokerResult != null) {
- if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) {
- this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr());
+ changeTransactionCheckServiceStatus(shouldStart);
+
+ if (this.ackMessageProcessor != null) {
+ LOG.info("Set PopReviveService Status to {}", shouldStart);
+ this.ackMessageProcessor.setPopReviveServiceStatus(shouldStart);
+ }
+ }
+
+ private synchronized void changeTransactionCheckServiceStatus(boolean shouldStart) {
+ if (isTransactionCheckServiceStart != shouldStart) {
+ LOG.info("TransactionCheckService status changed to {}", shouldStart);
+ if (shouldStart) {
+ this.transactionalMessageCheckService.start();
+ } else {
+ this.transactionalMessageCheckService.shutdown(true);
}
+ isTransactionCheckServiceStart = shouldStart;
+ }
+ }
- this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr());
+ public synchronized void changeScheduleServiceStatus(boolean shouldStart) {
+ if (isScheduleServiceStart != shouldStart) {
+ LOG.info("ScheduleServiceStatus changed to {}", shouldStart);
+ if (shouldStart) {
+ this.scheduleMessageService.start();
+ } else {
+ this.scheduleMessageService.stop();
+ }
+ isScheduleServiceStart = shouldStart;
- if (checkOrderConfig) {
- this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable());
+ if (timerMessageStore != null) {
+ timerMessageStore.syncLastReadTimeMs();
+ timerMessageStore.setShouldRunningDequeue(shouldStart);
}
}
}
+ public MessageStore getMessageStoreByBrokerName(String brokerName) {
+ if (this.brokerConfig.getBrokerName().equals(brokerName)) {
+ return this.getMessageStore();
+ }
+ return null;
+ }
+
+ public BrokerIdentity getBrokerIdentity() {
+ if (messageStoreConfig.isEnableDLegerCommitLog()) {
+ return new BrokerIdentity(
+ brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),
+ Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)), brokerConfig.isInBrokerContainer());
+ } else {
+ return new BrokerIdentity(
+ brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),
+ brokerConfig.getBrokerId(), brokerConfig.isInBrokerContainer());
+ }
+ }
+
public TopicConfigManager getTopicConfigManager() {
return topicConfigManager;
}
@@ -748,6 +2150,10 @@ public void setTopicConfigManager(TopicConfigManager topicConfigManager) {
this.topicConfigManager = topicConfigManager;
}
+ public TopicQueueMappingManager getTopicQueueMappingManager() {
+ return topicQueueMappingManager;
+ }
+
public String getHAServerAddr() {
return this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort();
}
@@ -760,10 +2166,18 @@ public SlaveSynchronize getSlaveSynchronize() {
return slaveSynchronize;
}
+ public ScheduledExecutorService getScheduledExecutorService() {
+ return scheduledExecutorService;
+ }
+
public ExecutorService getPullMessageExecutor() {
return pullMessageExecutor;
}
+ public ExecutorService getPutMessageFutureExecutor() {
+ return putMessageFutureExecutor;
+ }
+
public void setPullMessageExecutor(ExecutorService pullMessageExecutor) {
this.pullMessageExecutor = pullMessageExecutor;
}
@@ -772,8 +2186,8 @@ public BlockingQueue getSendThreadPoolQueue() {
return sendThreadPoolQueue;
}
- public FilterServerManager getFilterServerManager() {
- return filterServerManager;
+ public BlockingQueue getAckThreadPoolQueue() {
+ return ackThreadPoolQueue;
}
public BrokerStatsManager getBrokerStatsManager() {
@@ -786,7 +2200,7 @@ public List getSendMessageHookList() {
public void registerSendMessageHook(final SendMessageHook hook) {
this.sendMessageHookList.add(hook);
- log.info("register SendMessageHook Hook, {}", hook.hookName());
+ LOG.info("register SendMessageHook Hook, {}", hook.hookName());
}
public List getConsumeMessageHookList() {
@@ -795,11 +2209,12 @@ public List getConsumeMessageHookList() {
public void registerConsumeMessageHook(final ConsumeMessageHook hook) {
this.consumeMessageHookList.add(hook);
- log.info("register ConsumeMessageHook Hook, {}", hook.hookName());
+ LOG.info("register ConsumeMessageHook Hook, {}", hook.hookName());
}
public void registerServerRPCHook(RPCHook rpcHook) {
getRemotingServer().registerRPCHook(rpcHook);
+ this.fastRemotingServer.registerRPCHook(rpcHook);
}
public RemotingServer getRemotingServer() {
@@ -810,6 +2225,14 @@ public void setRemotingServer(RemotingServer remotingServer) {
this.remotingServer = remotingServer;
}
+ public CountDownLatch getRemotingServerStartLatch() {
+ return remotingServerStartLatch;
+ }
+
+ public void setRemotingServerStartLatch(CountDownLatch remotingServerStartLatch) {
+ this.remotingServerStartLatch = remotingServerStartLatch;
+ }
+
public void registerClientRPCHook(RPCHook rpcHook) {
this.getBrokerOuterAPI().registerRPCHook(rpcHook);
}
@@ -829,4 +2252,180 @@ public void setStoreHost(InetSocketAddress storeHost) {
public Configuration getConfiguration() {
return this.configuration;
}
+
+ public BlockingQueue getHeartbeatThreadPoolQueue() {
+ return heartbeatThreadPoolQueue;
+ }
+
+ public TransactionalMessageCheckService getTransactionalMessageCheckService() {
+ return transactionalMessageCheckService;
+ }
+
+ public void setTransactionalMessageCheckService(
+ TransactionalMessageCheckService transactionalMessageCheckService) {
+ this.transactionalMessageCheckService = transactionalMessageCheckService;
+ }
+
+ public TransactionalMessageService getTransactionalMessageService() {
+ return transactionalMessageService;
+ }
+
+ public void setTransactionalMessageService(TransactionalMessageService transactionalMessageService) {
+ this.transactionalMessageService = transactionalMessageService;
+ }
+
+ public AbstractTransactionalMessageCheckListener getTransactionalMessageCheckListener() {
+ return transactionalMessageCheckListener;
+ }
+
+ public void setTransactionalMessageCheckListener(
+ AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) {
+ this.transactionalMessageCheckListener = transactionalMessageCheckListener;
+ }
+
+ public BlockingQueue getEndTransactionThreadPoolQueue() {
+ return endTransactionThreadPoolQueue;
+
+ }
+
+ public Map getAccessValidatorMap() {
+ return accessValidatorMap;
+ }
+
+ public ExecutorService getSendMessageExecutor() {
+ return sendMessageExecutor;
+ }
+
+ public SendMessageProcessor getSendMessageProcessor() {
+ return sendMessageProcessor;
+ }
+
+ public QueryAssignmentProcessor getQueryAssignmentProcessor() {
+ return queryAssignmentProcessor;
+ }
+
+ public TopicQueueMappingCleanService getTopicQueueMappingCleanService() {
+ return topicQueueMappingCleanService;
+ }
+
+ public ExecutorService getAdminBrokerExecutor() {
+ return adminBrokerExecutor;
+ }
+
+ public BlockingQueue getLitePullThreadPoolQueue() {
+ return litePullThreadPoolQueue;
+ }
+
+ public ShutdownHook getShutdownHook() {
+ return shutdownHook;
+ }
+
+ public void setShutdownHook(ShutdownHook shutdownHook) {
+ this.shutdownHook = shutdownHook;
+ }
+
+ public long getMinBrokerIdInGroup() {
+ return this.brokerConfig.getBrokerId();
+ }
+
+ public BrokerController peekMasterBroker() {
+ return brokerConfig.getBrokerId() == MixAll.MASTER_ID ? this : null;
+ }
+
+ public BrokerMemberGroup getBrokerMemberGroup() {
+ return this.brokerMemberGroup;
+ }
+
+ public int getListenPort() {
+ return this.nettyServerConfig.getListenPort();
+ }
+
+ public List getBrokerAttachedPlugins() {
+ return brokerAttachedPlugins;
+ }
+
+ public EscapeBridge getEscapeBridge() {
+ return escapeBridge;
+ }
+
+ public long getShouldStartTime() {
+ return shouldStartTime;
+ }
+
+ public BrokerPreOnlineService getBrokerPreOnlineService() {
+ return brokerPreOnlineService;
+ }
+
+ public EndTransactionProcessor getEndTransactionProcessor() {
+ return endTransactionProcessor;
+ }
+
+ public boolean isScheduleServiceStart() {
+ return isScheduleServiceStart;
+ }
+
+ public boolean isTransactionCheckServiceStart() {
+ return isTransactionCheckServiceStart;
+ }
+
+ public ScheduleMessageService getScheduleMessageService() {
+ return scheduleMessageService;
+ }
+
+ public ReplicasManager getReplicasManager() {
+ return replicasManager;
+ }
+
+ public void setIsolated(boolean isolated) {
+ isIsolated = isolated;
+ }
+
+ public boolean isIsolated() {
+ return this.isIsolated;
+ }
+
+ public TimerCheckpoint getTimerCheckpoint() {
+ return timerCheckpoint;
+ }
+
+ public TopicRouteInfoManager getTopicRouteInfoManager() {
+ return this.topicRouteInfoManager;
+ }
+
+ public BlockingQueue getClientManagerThreadPoolQueue() {
+ return clientManagerThreadPoolQueue;
+ }
+
+ public BlockingQueue getConsumerManagerThreadPoolQueue() {
+ return consumerManagerThreadPoolQueue;
+ }
+
+ public BlockingQueue getAsyncPutThreadPoolQueue() {
+ return putThreadPoolQueue;
+ }
+
+ public BlockingQueue getReplyThreadPoolQueue() {
+ return replyThreadPoolQueue;
+ }
+
+ public BlockingQueue getAdminBrokerThreadPoolQueue() {
+ return adminBrokerThreadPoolQueue;
+ }
+
+ public ColdDataPullRequestHoldService getColdDataPullRequestHoldService() {
+ return coldDataPullRequestHoldService;
+ }
+
+ public void setColdDataPullRequestHoldService(
+ ColdDataPullRequestHoldService coldDataPullRequestHoldService) {
+ this.coldDataPullRequestHoldService = coldDataPullRequestHoldService;
+ }
+
+ public ColdDataCgCtrService getColdDataCgCtrService() {
+ return coldDataCgCtrService;
+ }
+
+ public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) {
+ this.coldDataCgCtrService = coldDataCgCtrService;
+ }
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
index 42c8da9f3fb..0b2f52f32eb 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
@@ -35,15 +35,40 @@ public static String getTopicConfigPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "topics.json";
}
+ public static String getTopicQueueMappingPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "topicQueueMapping.json";
+ }
+
public static String getConsumerOffsetPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "consumerOffset.json";
}
+ public static String getLmqConsumerOffsetPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "lmqConsumerOffset.json";
+ }
+
+ public static String getConsumerOrderInfoPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "consumerOrderInfo.json";
+ }
+
public static String getSubscriptionGroupPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json";
}
+ public static String getTimerCheckPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "timercheck";
+ }
+ public static String getTimerMetricsPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "timermetrics";
+ }
+ public static String getTransactionMetricsPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "transactionMetrics";
+ }
public static String getConsumerFilterPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "consumerFilter.json";
}
+
+ public static String getMessageRequestModePath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "messageRequestMode.json";
+ }
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPreOnlineService.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPreOnlineService.java
new file mode 100644
index 00000000000..de2ccb29399
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPreOnlineService.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+import org.apache.rocketmq.broker.schedule.DelayOffsetSerializeWrapper;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
+import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.ha.HAConnectionState;
+import org.apache.rocketmq.store.ha.HAConnectionStateNotificationRequest;
+import org.apache.rocketmq.store.timer.TimerCheckpoint;
+
+public class BrokerPreOnlineService extends ServiceThread {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ private final BrokerController brokerController;
+
+ private int waitBrokerIndex = 0;
+
+ public BrokerPreOnlineService(BrokerController brokerController) {
+ this.brokerController = brokerController;
+ }
+
+ @Override
+ public String getServiceName() {
+ if (this.brokerController != null && this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+ return brokerController.getBrokerIdentity().getIdentifier() + BrokerPreOnlineService.class.getSimpleName();
+ }
+ return BrokerPreOnlineService.class.getSimpleName();
+ }
+
+ @Override
+ public void run() {
+ LOGGER.info(this.getServiceName() + " service started");
+
+ while (!this.isStopped()) {
+ if (!this.brokerController.isIsolated()) {
+ LOGGER.info("broker {} is online", this.brokerController.getBrokerConfig().getCanonicalName());
+ break;
+ }
+ try {
+ boolean isSuccess = this.prepareForBrokerOnline();
+ if (!isSuccess) {
+ this.waitForRunning(1000);
+ } else {
+ break;
+ }
+ } catch (Exception e) {
+ LOGGER.error("Broker preOnline error, ", e);
+ }
+ }
+
+ LOGGER.info(this.getServiceName() + " service end");
+ }
+
+ CompletableFuture waitForHaHandshakeComplete(String brokerAddr) {
+ LOGGER.info("wait for handshake completion with {}", brokerAddr);
+ HAConnectionStateNotificationRequest request =
+ new HAConnectionStateNotificationRequest(HAConnectionState.TRANSFER, RemotingHelper.parseHostFromAddress(brokerAddr), true);
+ if (this.brokerController.getMessageStore().getHaService() != null) {
+ this.brokerController.getMessageStore().getHaService().putGroupConnectionStateRequest(request);
+ } else {
+ LOGGER.error("HAService is null, maybe broker config is wrong. For example, duplicationEnable is true");
+ request.getRequestFuture().complete(false);
+ }
+ return request.getRequestFuture();
+ }
+
+ private boolean futureWaitAction(boolean result, BrokerMemberGroup brokerMemberGroup) {
+ if (!result) {
+ LOGGER.error("wait for handshake completion failed, HA connection lost");
+ return false;
+ }
+ if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {
+ LOGGER.info("slave preOnline complete, start service");
+ long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+ this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+ }
+ return true;
+ }
+
+ private boolean prepareForMasterOnline(BrokerMemberGroup brokerMemberGroup) {
+ List brokerIdList = new ArrayList<>(brokerMemberGroup.getBrokerAddrs().keySet());
+ Collections.sort(brokerIdList);
+ while (true) {
+ if (waitBrokerIndex >= brokerIdList.size()) {
+ LOGGER.info("master preOnline complete, start service");
+ this.brokerController.startService(MixAll.MASTER_ID, this.brokerController.getBrokerAddr());
+ return true;
+ }
+
+ String brokerAddrToWait = brokerMemberGroup.getBrokerAddrs().get(brokerIdList.get(waitBrokerIndex));
+
+ try {
+ this.brokerController.getBrokerOuterAPI().
+ sendBrokerHaInfo(brokerAddrToWait, this.brokerController.getHAServerAddr(),
+ this.brokerController.getMessageStore().getBrokerInitMaxOffset(), this.brokerController.getBrokerAddr());
+ } catch (Exception e) {
+ LOGGER.error("send ha address to {} exception, {}", brokerAddrToWait, e);
+ return false;
+ }
+
+ CompletableFuture haHandshakeFuture = waitForHaHandshakeComplete(brokerAddrToWait)
+ .thenApply(result -> futureWaitAction(result, brokerMemberGroup));
+
+ try {
+ if (!haHandshakeFuture.get()) {
+ return false;
+ }
+ } catch (Exception e) {
+ LOGGER.error("Wait handshake completion exception, {}", e);
+ return false;
+ }
+
+ if (syncMetadataReverse(brokerAddrToWait)) {
+ waitBrokerIndex++;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ private boolean syncMetadataReverse(String brokerAddr) {
+ try {
+ LOGGER.info("Get metadata reverse from {}", brokerAddr);
+
+ String delayOffset = this.brokerController.getBrokerOuterAPI().getAllDelayOffset(brokerAddr);
+ DelayOffsetSerializeWrapper delayOffsetSerializeWrapper =
+ DelayOffsetSerializeWrapper.fromJson(delayOffset, DelayOffsetSerializeWrapper.class);
+
+ ConsumerOffsetSerializeWrapper consumerOffsetSerializeWrapper = this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(brokerAddr);
+
+ TimerCheckpoint timerCheckpoint = this.brokerController.getBrokerOuterAPI().getTimerCheckPoint(brokerAddr);
+
+ if (null != consumerOffsetSerializeWrapper && brokerController.getConsumerOffsetManager().getDataVersion().compare(consumerOffsetSerializeWrapper.getDataVersion()) <= 0) {
+ LOGGER.info("{}'s consumerOffset data version is larger than master broker, {}'s consumerOffset will be used.", brokerAddr, brokerAddr);
+ this.brokerController.getConsumerOffsetManager().getOffsetTable()
+ .putAll(consumerOffsetSerializeWrapper.getOffsetTable());
+ this.brokerController.getConsumerOffsetManager().getDataVersion().assignNewOne(consumerOffsetSerializeWrapper.getDataVersion());
+ this.brokerController.getConsumerOffsetManager().persist();
+ }
+
+ if (null != delayOffset && brokerController.getScheduleMessageService().getDataVersion().compare(delayOffsetSerializeWrapper.getDataVersion()) <= 0) {
+ LOGGER.info("{}'s scheduleMessageService data version is larger than master broker, {}'s delayOffset will be used.", brokerAddr, brokerAddr);
+ String fileName =
+ StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController
+ .getMessageStoreConfig().getStorePathRootDir());
+ try {
+ MixAll.string2File(delayOffset, fileName);
+ this.brokerController.getScheduleMessageService().load();
+ } catch (IOException e) {
+ LOGGER.error("Persist file Exception, {}", fileName, e);
+ }
+ }
+
+ if (null != this.brokerController.getTimerCheckpoint() && this.brokerController.getTimerCheckpoint().getDataVersion().compare(timerCheckpoint.getDataVersion()) <= 0) {
+ LOGGER.info("{}'s timerCheckpoint data version is larger than master broker, {}'s timerCheckpoint will be used.", brokerAddr, brokerAddr);
+ this.brokerController.getTimerCheckpoint().setLastReadTimeMs(timerCheckpoint.getLastReadTimeMs());
+ this.brokerController.getTimerCheckpoint().setMasterTimerQueueOffset(timerCheckpoint.getMasterTimerQueueOffset());
+ this.brokerController.getTimerCheckpoint().getDataVersion().assignNewOne(timerCheckpoint.getDataVersion());
+ this.brokerController.getTimerCheckpoint().flush();
+ }
+
+ for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {
+ if (brokerAttachedPlugin != null) {
+ brokerAttachedPlugin.syncMetadataReverse(brokerAddr);
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error("GetMetadataReverse Failed", e);
+ return false;
+ }
+
+ return true;
+ }
+
+ private boolean prepareForSlaveOnline(BrokerMemberGroup brokerMemberGroup) {
+ BrokerSyncInfo brokerSyncInfo;
+ try {
+ brokerSyncInfo = this.brokerController.getBrokerOuterAPI()
+ .retrieveBrokerHaInfo(brokerMemberGroup.getBrokerAddrs().get(MixAll.MASTER_ID));
+ } catch (Exception e) {
+ LOGGER.error("retrieve master ha info exception, {}", e);
+ return false;
+ }
+
+ if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0
+ && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {
+ LOGGER.info("Set master flush offset in slave to {}", brokerSyncInfo.getMasterFlushOffset());
+ this.brokerController.getMessageStore().setMasterFlushedOffset(brokerSyncInfo.getMasterFlushOffset());
+ }
+
+ if (brokerSyncInfo.getMasterHaAddress() != null) {
+ this.brokerController.getMessageStore().updateHaMasterAddress(brokerSyncInfo.getMasterHaAddress());
+ this.brokerController.getMessageStore().updateMasterAddress(brokerSyncInfo.getMasterAddress());
+ } else {
+ LOGGER.info("fetch master ha address return null, start service directly");
+ long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+ this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+ return true;
+ }
+
+ CompletableFuture haHandshakeFuture = waitForHaHandshakeComplete(brokerSyncInfo.getMasterHaAddress())
+ .thenApply(result -> futureWaitAction(result, brokerMemberGroup));
+
+ try {
+ if (!haHandshakeFuture.get()) {
+ return false;
+ }
+ } catch (Exception e) {
+ LOGGER.error("Wait handshake completion exception, {}", e);
+ return false;
+ }
+
+ return true;
+ }
+
+ private boolean prepareForBrokerOnline() {
+ BrokerMemberGroup brokerMemberGroup;
+ try {
+ brokerMemberGroup = this.brokerController.getBrokerOuterAPI().syncBrokerMemberGroup(
+ this.brokerController.getBrokerConfig().getBrokerClusterName(),
+ this.brokerController.getBrokerConfig().getBrokerName(),
+ this.brokerController.getBrokerConfig().isCompatibleWithOldNameSrv());
+ } catch (Exception e) {
+ LOGGER.error("syncBrokerMemberGroup from namesrv error, start service failed, will try later, ", e);
+ return false;
+ }
+
+ if (brokerMemberGroup != null && !brokerMemberGroup.getBrokerAddrs().isEmpty()) {
+ long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+
+ if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {
+ return prepareForMasterOnline(brokerMemberGroup);
+ } else if (minBrokerId == MixAll.MASTER_ID) {
+ return prepareForSlaveOnline(brokerMemberGroup);
+ } else {
+ LOGGER.info("no master online, start service directly");
+ this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+ }
+ } else {
+ LOGGER.info("no other broker online, will start service directly");
+ this.brokerController.startService(this.brokerController.getBrokerConfig().getBrokerId(), this.brokerController.getBrokerAddr());
+ }
+
+ return true;
+ }
+
+ private long getMinBrokerId(Map brokerAddrMap) {
+ Map