2.0 Support + Plugin Directory Support + Supporting Tests + Defaults Change

Author: Dale McDiarmid

.kitchen.yml

@@ -56,24 +56,66 @@ platforms:
       use_sudo: false
 
 suites:
-  - name: standard
+  - name: standard-2x
     provisioner:
       playbook: test/integration/standard.yml
     run_list:
     attributes:
-  - name: package
+  - name: package-2x
     run_list:
     attributes:
+      extra_vars:
+        es_plugins:
+          - plugin: lmenezes/elasticsearch-kopf
+            version: master
+          - plugin: license
+          - plugin: marvel-agent
     provisioner:
       playbook: test/integration/package.yml
-  - name: config
+  - name: config-2x
     run_list:
     attributes:
     provisioner:
       playbook: test/integration/config.yml
-  - name: multi
+  - name: multi-2x
     run_list:
     attributes:
     provisioner:
       playbook: test/integration/multi.yml
-
+  - name: standard-1x
+    provisioner:
+      playbook: test/integration/standard.yml
+    run_list:
+    attributes:
+      extra_vars:
+        es_major_version: 1.7
+        es_version: 1.7.3
+  - name: package-1x
+    run_list:
+    attributes:
+      extra_vars:
+        es_major_version: 1.7
+        es_version: 1.7.3
+        es_plugins:
+          - plugin: lmenezes/elasticsearch-kopf
+            version: master
+          - plugin: elasticsearch/marvel
+            version: latest
+    provisioner:
+      playbook: test/integration/package.yml
+  - name: config-1x
+    run_list:
+    attributes:
+      extra_vars:
+        es_major_version: 1.7
+        es_version: 1.7.3
+    provisioner:
+      playbook: test/integration/config.yml
+  - name: multi-1x
+    run_list:
+    attributes:
+      extra_vars:
+        es_major_version: 1.7
+        es_version: 1.7.3
+    provisioner:
+      playbook: test/integration/multi.yml

defaults/main.yml

@@ -1,6 +1,6 @@
 ---
-es_major_version: "1.7"
-es_version: "1.7.3"
+es_major_version: "2.x"
+es_version: "2.1.0"
 es_version_lock: false
 es_use_repository: true
 es_start_service: true
@@ -14,4 +14,5 @@ es_config: {}
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dir: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
-es_work_dir: "/tmp/elasticsearch"
+es_work_dir: "/tmp/elasticsearch"
+es_plugin_dir: "/usr/share/elasticsearch/plugins"

handlers/main.yml

@@ -1,4 +1,4 @@
 
 - name: restart elasticsearch
   service: name={{instance_init_script | basename}} state=restarted enabled=yes
-  when: es_start_service and not elasticsearch_started.changed
+  when: es_start_service and (not elasticsearch_started.changed or plugin_installed.changed)

tasks/checkParameters.yml

@@ -3,15 +3,15 @@
 - fail: msg="es_instance_name must be specified"
   when: es_instance_name is not defined
 
-- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or es_config['discovery.zen.ping.multicast.enabled']}}
+- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}}
 
-- fail: msg="Parameter 'http.port' must be defined when multicast is disabled"
+- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled"
   when: not multi_cast and es_config['http.port'] is not defined
 
-- fail: msg="Parameter 'transport.tcp.port' must be defined when multicast is disabled"
+- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled"
   when: not multi_cast and es_config['transport.tcp.port'] is not defined
 
-- fail: msg="Parameter 'discovery.zen.ping.unicast.hosts' must be defined when multicast is disabled"
+- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled"
   when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined
 
 #If the user attempts to lock memory they must specify a heap size

tasks/elasticsearch-config.yml

@@ -12,7 +12,7 @@
 
 #For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
 
-- set_fact: pid_dir={{es_pid_dir}}/{{inventory_hostname}}-{{ es_instance_name }}
+- set_fact: pid_dir={{ es_pid_dir }}/{{inventory_hostname}}-{{ es_instance_name }}
 
 - set_fact: data_dir={{ es_data_dir }}/{{inventory_hostname}}-{{ es_instance_name }}
 
@@ -34,11 +34,14 @@
   file: state=directory path={{ log_dir }} owner={{ es_user }} group={{ es_group }}
 
 - name: Create Config Directory
-  file: path={{ instance_config_directory }} state=directory owner={{ es_user }} group={{ es_group }}
+  file: path={{ conf_dir }} state=directory owner={{ es_user }} group={{ es_group }}
+
+- name: Create Plugin Directory
+  file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }}
 
 #Copy the config template
 - name: Copy Configuration File
-  template: src=elasticsearch.yml.j2 dest={{instance_config_directory}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=elasticsearch.yml.j2 dest={{conf_dir}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
 
 #Copy the instance specific default file
@@ -64,5 +67,5 @@
 
 #Copy the logging.yml
 - name: Copy Logging.yml File for Instance
-  template: src=logging.yml.j2 dest={{instance_config_directory}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch

tasks/elasticsearch-plugins.yml

@@ -1,15 +1,26 @@
 ---
+
 - name: Remove elasticsearch plugins
-  command: /usr/share/elasticsearch/bin/plugin --remove {{ item.plugin }} --silent
+  command: "{{es_home}}/bin/plugin remove {{ item.plugin }} --silent"
   ignore_errors: yes
   with_items: es_plugins
   when: ( ansible_os_family == 'RedHat' or ansible_os_family == 'Debian' ) and es_plugins_reinstall
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
 
 - name: Install elasticsearch plugins
   #debug: var=item
-  command: /usr/share/elasticsearch/bin/plugin --install {{ item.plugin }}/{{ item.version }} --silent
-  register: command_result
-  failed_when: "'Failed to install' in command_result.stderr"
-  changed_when: command_result.rc == 0
+  command: "{{es_home}}/bin/plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} --silent"
+  register: plugin_installed
+  failed_when: "'Failed to install' in plugin_installed.stderr"
+  changed_when: plugin_installed.rc == 0
   with_items: es_plugins
-  when: ( ansible_os_family == 'RedHat' or ansible_os_family == 'Debian' )
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'Debian'
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+
+#Set permissions on plugins directory
+- name: Set Plugin Directory Permissions
+  file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/elasticsearch.yml

@@ -2,7 +2,8 @@
 
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
-- set_fact: instance_config_directory={{ es_conf_dir }}/{{es_instance_name}}
+- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
+- set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}}
 - set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
 
 - debug: msg="Node configuration {{ es_config }} "
@@ -20,7 +21,6 @@
 - name: Elasticsearch configuration
   include: elasticsearch-config.yml
 
-
 # Make sure the service is started, and restart if necessary
 - name: Start elasticsearch service
   service: name={{instance_init_script | basename}} state=started enabled=yes

templates/elasticsearch.j2

@@ -6,10 +6,7 @@
 ES_HOME={{es_home}}
 
 # Elasticsearch configuration directory
-CONF_DIR={{instance_config_directory}}
-
-# Elasticsearch configuration file
-CONF_FILE={{instance_config_directory}}/elasticsearch.yml
+CONF_DIR={{conf_dir}}
 
 # Elasticsearch data directory
 DATA_DIR={{data_dir}}

templates/elasticsearch.repo

@@ -1,5 +1,5 @@
 [elasticsearch-{{ es_major_version }}]
-name=Elasticsearch repository for {{ es_major_version }}.x packages
+name=Elasticsearch repository for {{ es_major_version }} packages
 baseurl=http://packages.elastic.co/elasticsearch/{{ es_major_version }}/centos
 gpgcheck=1
 gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch

templates/elasticsearch.yml.j2

@@ -14,10 +14,12 @@ node.name: {{inventory_hostname}}-{{es_instance_name}}
 #################################### Paths ####################################
 
 # Path to directory containing configuration (this file and logging.yml):
-path.conf: {{ instance_config_directory }}
+path.conf: {{ conf_dir }}
 
 path.data: {{ data_dir }}
 
 path.work: {{ work_dir }}
 
-path.logs: {{ log_dir }}
+path.logs: {{ log_dir }}
+
+path.plugins: {{ plugin_dir }}

templates/init/debian/elasticsearch.j2

@@ -26,6 +26,7 @@ DESC="Elasticsearch Server - {{es_config['node.name']}}"
 {% else %}
 DESC="Elasticsearch Server - {{es_instance_name}}"
 {% endif %}
+
 DEFAULT=/etc/default/$NAME
 
 if [ `id -u` -ne 0 ]; then
@@ -40,23 +41,13 @@ if [ -r /etc/default/rcS ]; then
 	. /etc/default/rcS
 fi
 
+
 # The following variables can be overwritten in $DEFAULT
 
 # Run Elasticsearch as this user ID and group ID
 ES_USER=elasticsearch
 ES_GROUP=elasticsearch
 
-# The first existing directory is used for JAVA_HOME (if JAVA_HOME is not defined in $DEFAULT)
-JDK_DIRS="/usr/lib/jvm/java-8-oracle/ /usr/lib/jvm/j2sdk1.8-oracle/ /usr/lib/jvm/jdk-7-oracle-x64 /usr/lib/jvm/java-7-oracle /usr/lib/jvm/j2sdk1.7-oracle/ /usr/lib/jvm/java-7-openjdk /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-armhf /usr/lib/jvm/java-7-openjdk-i386/ /usr/lib/jvm/default-java"
-
-# Look for the right JVM to use
-for jdir in $JDK_DIRS; do
-    if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then
-        JAVA_HOME="$jdir"
-    fi
-done
-export JAVA_HOME
-
 # Directory where the Elasticsearch binary distribution resides
 ES_HOME=/usr/share/$NAME
 
@@ -85,15 +76,9 @@ LOG_DIR=/var/log/$NAME
 # Elasticsearch data directory
 DATA_DIR=/var/lib/$NAME
 
-# Elasticsearch work directory
-WORK_DIR=/tmp/$NAME
-
 # Elasticsearch configuration directory
 CONF_DIR=/etc/$NAME
 
-# Elasticsearch configuration file (elasticsearch.yml)
-CONF_FILE=$CONF_DIR/elasticsearch.yml
-
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 MAX_MAP_COUNT=262144
 
@@ -110,15 +95,23 @@ if [ -f "$DEFAULT" ]; then
 	. "$DEFAULT"
 fi
 
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
-DAEMON_OPTS="-d -p $PID_FILE --default.config=$CONF_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.work=$WORK_DIR --default.path.conf=$CONF_DIR"
+DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
 
 export ES_HEAP_SIZE
 export ES_HEAP_NEWSIZE
 export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
+export ES_GC_LOG_FILE
+export JAVA_HOME
 
 # Check DAEMON exists
 test -x $DAEMON || exit 0
@@ -155,15 +148,15 @@ case "$1" in
 	fi
 
 	# Prepare environment
-	mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR"
+	mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR"
 
-    # Ensure that the PID_DIR exists (it is cleaned at OS startup time)
-    if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
-        mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
-    fi
-    if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then
-        touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
-    fi
+	# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
+	if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
+		mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
+	fi
+	if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then
+		touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
+	fi
 
 	if [ -n "$MAX_OPEN_FILES" ]; then
 		ulimit -n $MAX_OPEN_FILES
@@ -178,10 +171,9 @@ case "$1" in
 	fi
 
 	# Start Daemon
-	start-stop-daemon --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
+	start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
 	return=$?
-	if [ $return -eq 0 ]
-	then
+	if [ $return -eq 0 ]; then
 		i=0
 		timeout=10
 		# Wait for the process to be properly started before exiting
@@ -194,17 +186,18 @@ case "$1" in
 				exit 1
 			fi
 		done
-	else
-		log_end_msg $return
 	fi
-	;;		
+	log_end_msg $return
+	exit $return
+	;;
   stop)
 	log_daemon_msg "Stopping $DESC"
 
-	if [ -f "$PID_FILE" ]; then 
+	if [ -f "$PID_FILE" ]; then
 		start-stop-daemon --stop --pidfile "$PID_FILE" \
 			--user "$ES_USER" \
-			--retry=TERM/20/KILL/5 >/dev/null
+			--quiet \
+			--retry forever/TERM/20 > /dev/null
 		if [ $? -eq 1 ]; then
 			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
 		elif [ $? -eq 3 ]; then
@@ -220,7 +213,7 @@ case "$1" in
 	;;
   status)
 	status_of_proc -p $PID_FILE elasticsearch elasticsearch && exit 0 || exit $?
-    ;;
+	;;
   restart|force-reload)
 	if [ -f "$PID_FILE" ]; then
 		$0 stop
@@ -234,4 +227,4 @@ case "$1" in
 	;;
 esac
 
-exit 0
+exit 0