From bc3fa8be61cda430288881a53ce45155b550653c Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Wed, 13 Jan 2021 07:52:59 +0800 Subject: [PATCH 1/6] ET-918 --- defaults/main.yml | 3 +++ tasks/elasticsearch-config.yml | 1 + templates/jvm.options.j2 | 2 +- test/integration/xpack-upgrade-trial | 1 - vars/main.yml | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) delete mode 120000 test/integration/xpack-upgrade-trial diff --git a/defaults/main.yml b/defaults/main.yml index 6c1c0017..ca31adda 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -69,3 +69,6 @@ es_ssl_verification_mode: "certificate" es_validate_certs: "yes" es_delete_unmanaged_file: true es_delete_unmanaged_native: true + +es_gc_log_file_count: 5 +es_gc_log_file_size: 64m \ No newline at end of file diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 36d928b3..efb0b7b3 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -33,6 +33,7 @@ with_items: - "{{ es_log_dir }}" - "{{ es_data_dirs }}" + - "{{ es_tmp_dir }}" #Copy the config template - name: Copy Configuration File diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 index 3acf3988..e326a96b 100644 --- a/templates/jvm.options.j2 +++ b/templates/jvm.options.j2 @@ -144,7 +144,7 @@ 8:-XX:GCLogFileSize=64m # JDK 9+ GC logging -9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_log_dir }}/gc.log:utctime,pid,tags:filecount=32,filesize=64m +9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_log_dir }}/gc.log:utctime,pid,tags:filecount={{es_gc_log_file_count}},filesize={{es_gc_log_file_size}} {% if es_version is version('7.5.0', '<') %} # due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise # time/date parsing will break in an incompatible way for some date patterns and locals diff --git a/test/integration/xpack-upgrade-trial b/test/integration/xpack-upgrade-trial deleted file mode 120000 index 3021ce03..00000000 --- a/test/integration/xpack-upgrade-trial +++ /dev/null @@ -1 +0,0 @@ -xpack-upgrade \ No newline at end of file diff --git a/vars/main.yml b/vars/main.yml index 91b7b699..e19d276b 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,4 +1,4 @@ --- es_package_url: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch" -reserved_xpack_users: ["elastic","kibana","logstash_system"] +reserved_xpack_users: ["elastic","kibana_system","logstash_system"] sysd_config_file: "/etc/systemd/system/elasticsearch.service.d/override.conf" From c4b05903dadf7586114c050486ade3df21e97016 Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Wed, 20 Jan 2021 21:59:47 +0800 Subject: [PATCH 2/6] Added a task to upload a predefined log4j.properties in "files" instead of "templates" directory. Added a new var es_config_log4j2_file. --- tasks/elasticsearch-config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index efb0b7b3..de102392 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -104,3 +104,14 @@ force: yes notify: restart elasticsearch when: es_config_log4j2 != '' + +- name: Upload log4j2.properties File from files/ + become: yes + copy: + src: "{{ es_config_log4j2_file }}" + dest: "{{ es_conf_dir }}/log4j2.properties" + owner: root + group: "{{ es_group }}" + mode: "660" + notify: restart elasticsearch + when: es_config_log4j2_file != '' \ No newline at end of file From 359cb8a08104b304f67a1152b067db9baaab6955 Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Sun, 7 Feb 2021 09:48:54 +0800 Subject: [PATCH 3/6] ET;918 ET-912 added support to manage and load component and index templates v2. --- tasks/elasticsearch-template.yml | 63 ++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index e7669324..7c3e4e13 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -29,3 +29,66 @@ with_fileglob: - "{{ es_templates_fileglob | default('') }}" run_once: True + + +- name: ensure component templates dir is created + file: + path: "{{ es_conf_dir }}/templates/component_templates" + state: directory + owner: root + group: "{{ es_group }}" + mode: "2750" + when: "es_component_templates_fileglob is defined and (es_component_templates_fileglob | length > 0)" + +- name: Copy component templates to elasticsearch + copy: src={{ item }} dest={{ es_conf_dir }}/templates/component_templates owner=root group={{ es_group }} mode=0660 + register: load_component_templates + with_fileglob: + - "{{ es_component_templates_fileglob | default('') }}" + +- name: Install component templates + uri: + url: "{{ es_api_uri }}/_component_template/{{item | filename}}" + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username | default(omit)}}" + password: "{{es_api_basic_auth_password | default(omit)}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', item) }}" + validate_certs: "{{ es_validate_certs }}" + when: load_component_templates.changed and es_start_service + with_fileglob: + - "{{ es_component_templates_fileglob | default('') }}" + run_once: True + +- name: ensure index templates dir is created + file: + path: "{{ es_conf_dir }}/templates/index_templates" + state: directory + owner: root + group: "{{ es_group }}" + mode: "2750" + when: "es_index_templates_fileglob is defined and (es_index_templates_fileglob | length > 0)" + +- name: Copy index templates to elasticsearch + copy: src={{ item }} dest={{ es_conf_dir }}/templates/index_templates owner=root group={{ es_group }} mode=0660 + register: load_index_templates + with_fileglob: + - "{{ es_index_templates_fileglob | default('') }}" + +- name: Install index templates + uri: + url: "{{ es_api_uri }}/_index_template/{{item | filename}}" + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username | default(omit)}}" + password: "{{es_api_basic_auth_password | default(omit)}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', item) }}" + validate_certs: "{{ es_validate_certs }}" + when: load_index_templates.changed and es_start_service + with_fileglob: + - "{{ es_index_templates_fileglob | default('') }}" + run_once: True \ No newline at end of file From 338af678e2340a5eb341b1109d9a9f11e8ca9a32 Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Sun, 7 Feb 2021 09:51:29 +0800 Subject: [PATCH 4/6] ET-918 ET-912 added missing comonent and index templates fileglob var. --- defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index d53f4939..62dccf53 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -7,6 +7,8 @@ es_version_lock: false es_use_repository: true es_add_repository: true es_templates_fileglob: "" +es_component_templates_fileglob: "" +es_index_templates_fileglob: "" es_repo_base: "https://artifacts.elastic.co" es_apt_key: "{{ es_repo_base }}/GPG-KEY-elasticsearch" es_apt_url: "deb {{ es_repo_base }}/packages/{{ es_repo_name }}/apt stable main" From 792751f58f79888f42e95ab6b9fc88fdbed8f82b Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Tue, 9 Feb 2021 11:35:09 +0800 Subject: [PATCH 5/6] Reverted changes for ET-918 ET-912. Removed support for v2 index templates. Moved to its own ansible-elasticsetup role. --- defaults/main.yml | 2 - tasks/elasticsearch-template.yml | 63 -------------------------------- 2 files changed, 65 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 62dccf53..d53f4939 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -7,8 +7,6 @@ es_version_lock: false es_use_repository: true es_add_repository: true es_templates_fileglob: "" -es_component_templates_fileglob: "" -es_index_templates_fileglob: "" es_repo_base: "https://artifacts.elastic.co" es_apt_key: "{{ es_repo_base }}/GPG-KEY-elasticsearch" es_apt_url: "deb {{ es_repo_base }}/packages/{{ es_repo_name }}/apt stable main" diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index 7c3e4e13..e7669324 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -29,66 +29,3 @@ with_fileglob: - "{{ es_templates_fileglob | default('') }}" run_once: True - - -- name: ensure component templates dir is created - file: - path: "{{ es_conf_dir }}/templates/component_templates" - state: directory - owner: root - group: "{{ es_group }}" - mode: "2750" - when: "es_component_templates_fileglob is defined and (es_component_templates_fileglob | length > 0)" - -- name: Copy component templates to elasticsearch - copy: src={{ item }} dest={{ es_conf_dir }}/templates/component_templates owner=root group={{ es_group }} mode=0660 - register: load_component_templates - with_fileglob: - - "{{ es_component_templates_fileglob | default('') }}" - -- name: Install component templates - uri: - url: "{{ es_api_uri }}/_component_template/{{item | filename}}" - method: PUT - status_code: 200 - user: "{{es_api_basic_auth_username | default(omit)}}" - password: "{{es_api_basic_auth_password | default(omit)}}" - force_basic_auth: yes - body_format: json - body: "{{ lookup('file', item) }}" - validate_certs: "{{ es_validate_certs }}" - when: load_component_templates.changed and es_start_service - with_fileglob: - - "{{ es_component_templates_fileglob | default('') }}" - run_once: True - -- name: ensure index templates dir is created - file: - path: "{{ es_conf_dir }}/templates/index_templates" - state: directory - owner: root - group: "{{ es_group }}" - mode: "2750" - when: "es_index_templates_fileglob is defined and (es_index_templates_fileglob | length > 0)" - -- name: Copy index templates to elasticsearch - copy: src={{ item }} dest={{ es_conf_dir }}/templates/index_templates owner=root group={{ es_group }} mode=0660 - register: load_index_templates - with_fileglob: - - "{{ es_index_templates_fileglob | default('') }}" - -- name: Install index templates - uri: - url: "{{ es_api_uri }}/_index_template/{{item | filename}}" - method: PUT - status_code: 200 - user: "{{es_api_basic_auth_username | default(omit)}}" - password: "{{es_api_basic_auth_password | default(omit)}}" - force_basic_auth: yes - body_format: json - body: "{{ lookup('file', item) }}" - validate_certs: "{{ es_validate_certs }}" - when: load_index_templates.changed and es_start_service - with_fileglob: - - "{{ es_index_templates_fileglob | default('') }}" - run_once: True \ No newline at end of file From 0db64e9a74ad27f08e97bb5e837e493fe2cca332 Mon Sep 17 00:00:00 2001 From: Hendry Lim Date: Tue, 16 Feb 2021 10:10:23 +0800 Subject: [PATCH 6/6] ET-918 ET-912 fixed error (GET with request body) when retrieving native role list due to body_format attribute was specified. --- tasks/xpack/security/elasticsearch-security-native.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/xpack/security/elasticsearch-security-native.yml b/tasks/xpack/security/elasticsearch-security-native.yml index 0d04a280..f8502640 100644 --- a/tasks/xpack/security/elasticsearch-security-native.yml +++ b/tasks/xpack/security/elasticsearch-security-native.yml @@ -137,7 +137,6 @@ uri: url: "{{ es_api_uri }}/{{ es_security_api }}/role" method: GET - body_format: json user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes