Enabled ElasticSearch 6 support

Author: paliarush

.travis.yml

@@ -37,3 +37,5 @@ after_script:
   - sudo kubectl logs -l app=magento -c monolith
   - sudo kubectl logs -l app=magento -c nginx
   - sudo cat /etc/exports
+  - sudo cat ./tests/logs/*
+  - sudo cat ./logs/*

README.md

@@ -63,6 +63,7 @@ If you never used Kubernetes before, read the [Kubernetes Docs](https://kubernet
 
 The software listed below should be available in [PATH](https://en.wikipedia.org/wiki/PATH_\(variable\)) (except for PHP Storm).
 
+- [Docker](https://docs.docker.com/docker-for-mac/install/)
 - [Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/)
 - [Helm](https://docs.helm.sh/using_helm/#installing-helm)
 - [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
@@ -343,14 +344,9 @@ Use the following commands to enable/disable varnish without reinstalling Magent
 
 ### Activating ElasticSearch
 
-Not available yet.
-<!--
-:information_source: Available in Magento EE only.
-
 Set `search_engine: "elasticsearch"` in [config.yaml](etc/config.yaml.dist) to use ElasticSearch as current search engine or `search_engine: "mysql"` to use MySQL. Changes will be applied on `m-reinstall`.
 
 Use the following commands to switch between search engines without reinstalling Magento: `m-search-engine elasticsearch` or `m-search-engine mysql`.
--->
 
 ### Redis for caching
 

etc/.gitignore

@@ -1,11 +1 @@
-/*
-!/.gitignore
-!/docker
-!/composer
-/composer/*
-!/composer/auth.json.dist
-!/config.yaml.dist
-!/magento2_virtual_host.conf.dist
-!/guest
-!/kubernetes
-!/helm
+config.yaml

etc/config.yaml.dist

@@ -42,7 +42,7 @@ environment:
   # [To apply changes: m-reinstall OR m-varnish enable/disable]
   use_varnish: 0
   # [To apply changes: m-reinstall] Possible values: mysql, elasticsearch
-  search_engine: "mysql"
+  search_engine: "elasticsearch"
 
 magento:
   # [To apply changes: m-switch-to-ce -f OR m-switch-to-ee -f]

etc/helm/charts/elasticsearch-6.5.0.tgz

@@ -1,12 +1,15 @@
 dependencies:
 - name: mysql
   repository: https://kubernetes-charts.storage.googleapis.com
-  version: 0.9.2
+  version: 0.15.0
 - name: redis
   repository: https://kubernetes-charts.storage.googleapis.com
-  version: 5.0.0
+  version: 6.4.5
 - name: rabbitmq
   repository: https://kubernetes-charts.storage.googleapis.com
-  version: 3.6.3
-digest: sha256:c75abdd584e0aa2ac7d476f1641aa9fe1f18a9526114a0b5a075266ee62fb47f
-generated: 2019-01-31T20:07:53.996726-06:00
+  version: 5.5.0
+- name: elasticsearch
+  repository: https://helm.elastic.co
+  version: 6.5.0
+digest: sha256:f5f413c09690010e1ed93c9fbc0f268ea869eca6e1d42e83729eeee70cdb1bfb
+generated: 2019-05-01T14:43:50.018673-05:00

etc/helm/charts/mysql-0.15.0.tgz

@@ -1,14 +1,20 @@
 dependencies:
 - name: mysql
-  version: "0.9.2"
+  version: "0.15.0"
   appVersion: 5.7.14
   repository: "@stable"
 - name: redis
   repository: "@stable"
-  version: "5.0"
+  appVersion: "4.0.14"
+  version: "6.4.5"
 - name: rabbitmq
   repository: "@stable"
-  version: "3.6.3"
+  appVersion: "3.7.14"
+  version: "5.5.0"
+- name: elasticsearch
+  repository: "https://helm.elastic.co"
+  appVersion: "6.7.1"
+  version: "6.5.0"
 - name: checkout
   repository: file://./charts/
   version: 0.1.0

etc/helm/charts/mysql-0.9.2.tgz

@@ -85,3 +85,33 @@ rabbitmq:
   rabbitmq:
     username: admin
     password: 123123q
+
+elasticsearch:
+  imageTag: "6.7.1"
+  esMajorVersion: 6
+
+  replicas: 2
+  minimumMasterNodes: 1
+
+  # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  # Shrink default JVM heap.
+  esJavaOpts: "-Xmx128m -Xms128m"
+
+  # Allocate smaller chunks of memory per pod.
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "512M"
+    limits:
+      cpu: "1000m"
+      memory: "512M"
+
+  # Request smaller persistent volumes.
+  volumeClaimTemplate:
+    accessModes: [ "ReadWriteOnce" ]
+    storageClassName: "standard"
+    resources:
+      requests:
+        storage: 100M

etc/helm/charts/rabbitmq-3.6.3.tgz

@@ -173,26 +173,26 @@ if [[ ${force_project_cleaning} -eq 1 ]]; then
 #    fi
 
 
-#    if [[ $(isMinikubeRunning) -eq 1 ]]; then
-#        minikube stop 2> >(logError) | {
-#          while IFS= read -r line
-#          do
-#            filterVagrantOutput "${line}"
-#            lastline="${line}"
-#          done
-#          filterVagrantOutput "${lastline}"
-#        }
-#    fi
-#    if [[ $(isMinikubeStopped) -eq 1 ]]; then
-#        minikube delete 2> >(logError) | {
-#          while IFS= read -r line
-#          do
-#            filterVagrantOutput "${line}"
-#            lastline="${line}"
-#          done
-#          filterVagrantOutput "${lastline}"
-#        }
-#    fi
+    if [[ $(isMinikubeRunning) -eq 1 ]]; then
+        minikube stop 2> >(logError) | {
+          while IFS= read -r line
+          do
+            filterVagrantOutput "${line}"
+            lastline="${line}"
+          done
+          filterVagrantOutput "${lastline}"
+        }
+    fi
+    if [[ $(isMinikubeStopped) -eq 1 ]]; then
+        minikube delete 2> >(logError) | {
+          while IFS= read -r line
+          do
+            filterVagrantOutput "${line}"
+            lastline="${line}"
+          done
+          filterVagrantOutput "${lastline}"
+        }
+    fi
 
     mv "${vagrant_dir}/etc/guest/.gitignore" "${vagrant_dir}/etc/.gitignore.back"
     rm -rf "${vagrant_dir}/.vagrant" "${vagrant_dir}/etc/guest"

etc/helm/charts/rabbitmq-5.5.0.tgz

@@ -15,6 +15,7 @@ if [[ ${debug_vagrant_project} -eq 1 ]]; then
 fi
 
 cd "${vagrant_dir}"
-vagrant ssh -c "bash /vagrant/scripts/guest/m-search-engine ${@}" 2> >(logError)
+
+executeInMagento2Container -- "${vagrant_dir}/scripts/guest/m-search-engine" ${@} 2> >(logError)
 
 info "$(regular)See details in $(bold)${vagrant_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:vagrant_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)"

etc/helm/charts/redis-5.0.0.tgz

@@ -12,27 +12,23 @@ if [[ -z "$(grep "search_engine:" ${vagrant_dir}/etc/config.yaml)" ]]; then
     sed -i '/environment:/a \ \ search_engine: "mysql"' ${vagrant_dir}/etc/config.yaml
 fi
 
-if [[ -f "${MAGENTO_ROOT}/LICENSE_EE.txt" ]]; then
-    search_engine="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_search_engine")"
-    if [[ ${search_engine} == "elasticsearch" ]]; then
-        status "Configuring ElasticSearch search engine"
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/engine' 'elasticsearch'
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch_server_hostname' 'localhost'
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch_server_port' '9200'
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch_index_prefix' 'magento2'
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch_enable_auth' '0'
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch_server_timeout' '15'
-    elif [[ ${search_engine} == "mysql" ]]; then
-        status "Configuring MySQL search engine"
-        "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/engine' 'mysql'
-    fi
-
-    bash "${VAGRANT_ROOT}/scripts/guest/m-clear-cache"
-
-    status "Rebuilding catalog search index"
-    php "${MAGENTO_ROOT}/bin/magento" indexer:reindex catalogsearch_fulltext 2> >(logError) > >(log)
-else
-    status "Search engine configuration was not changed (custom search engine is not available in Magento CE)."
+search_engine="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "environment_search_engine")"
+if [[ ${search_engine} == "elasticsearch" ]]; then
+    status "Configuring ElasticSearch search engine"
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/engine' 'elasticsearch6'
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch6_server_hostname' 'elasticsearch-master'
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch6_server_port' '9200'
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch6_index_prefix' 'magento2'
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch6_enable_auth' '0'
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/elasticsearch6_server_timeout' '15'
+elif [[ ${search_engine} == "mysql" ]]; then
+    status "Configuring MySQL search engine"
+    "${VAGRANT_ROOT}/scripts/guest/update_magento_config" 'catalog/search/engine' 'mysql'
 fi
 
+bash "${VAGRANT_ROOT}/scripts/guest/m-clear-cache"
+
+status "Rebuilding catalog search index"
+php "${MAGENTO_ROOT}/bin/magento" indexer:reindex catalogsearch_fulltext 2> >(logError) > >(log)
+
 decrementNestingLevel

etc/helm/charts/redis-6.4.5.tgz

@@ -7,8 +7,9 @@ source "${vagrant_dir}/scripts/functions.sh"
 status "Checking requirements"
 incrementNestingLevel
 
+nfs_enabled="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs")"
 nfs_exports_record="$(bash "${vagrant_dir}/scripts/host/get_nfs_exports_record.sh")"
-if [[ -z "$(grep "${nfs_exports_record}" /etc/exports)" ]]; then
+if [[ ${nfs_enabled} -eq 1 ]] && [[ -z "$(grep "${nfs_exports_record}" /etc/exports)" ]]; then
     warning "NFS exports configuration required on the host. Please execute 'bash ${vagrant_dir}/scripts/host/configure_nfs_exports.sh' first."
     exit 1
 fi

etc/helm/requirements.lock

@@ -14,6 +14,18 @@ if [[ ${debug_vagrant_project} -eq 1 ]]; then
     set -x
 fi
 
+nfs_enabled="$(bash "${vagrant_dir}/scripts/get_config_value.sh" "guest_use_nfs")"
+if [[ ${nfs_enabled} -eq 0 ]]; then
+    status "Skipping NFS configuration per config.yaml"
+    exit 0
+fi
+
+userId=$(id -u)
+if [[ ${userId} -eq 0 ]]; then
+    error "${vagrant_dir}/scripts/host/configure_nfs_exports.sh MUST NOT be run with sudo. Run it from unprivileged user and enter password when prompted."
+    exit 1
+fi
+
 host_os="$(bash "${vagrant_dir}/scripts/host/get_host_os.sh")"
 nfs_exports_record="$(bash "${vagrant_dir}/scripts/host/get_nfs_exports_record.sh")"
 if [[ ${host_os} == "OSX" ]]; then

etc/helm/requirements.yaml

@@ -19,7 +19,7 @@ helm list -q | xargs helm delete --purge 2>/dev/null
 set -e
 
 # TODO: Need to make sure all resources have been successfully deleted before the attempt of recreating them
-sleep 7
+#sleep 20
 enable_nfs="true"
 enable_checkout="false"
 while getopts 'de' flag; do

etc/helm/values.yaml

@@ -33,6 +33,10 @@ function executeExtendedCommonAssertions()
     # Test search
     createSimpleProduct
     assertSearchWorks
+
+    assertElasticSearchEnabled
+    assertElasticSearchDisablingWorks
+    assertElasticSearchEnablingWorks
 }
 
 ## Assertions
@@ -292,11 +296,13 @@ function assertElasticSearchEnabled()
     echo "## assertElasticSearchEnabled" >>${current_log_file_path}
 
     cd "${vagrant_dir}"
-    elasticSearchHealth="$(vagrant ssh -c 'curl -i http://127.0.0.1:9200/_cluster/health')"
+
+    elasticSearchHealth="$(executeInMagento2Container curl -- '-i' 'http://elasticsearch-master:9200/_cluster/health')"
+
     assertTrue "ElasticSearch server is down:
         ${elasticSearchHealth}" '[[ ${elasticSearchHealth} =~ \"status\":\"(green|yellow)\" ]]'
 
-    listOfIndexes="$(vagrant ssh -c 'curl -i http://127.0.0.1:9200/_cat/indices?v')"
+    listOfIndexes="$(executeInMagento2Container curl -- '-i' 'http://elasticsearch-master:9200/_cat/indices?v')"
     assertTrue "Products index is not available in ElasticSearch:
         ${listOfIndexes}" '[[ ${listOfIndexes} =~ magento2_product ]]'
 
@@ -309,11 +315,11 @@ function assertElasticSearchDisabled()
     echo "## assertElasticSearchDisabled" >>${current_log_file_path}
 
     cd "${vagrant_dir}"
-    elasticSearchHealth="$(vagrant ssh -c 'curl -i http://127.0.0.1:9200/_cluster/health')"
+    elasticSearchHealth="$(executeInMagento2Container curl -- '-i' 'http://elasticsearch-master:9200/_cluster/health')"
     assertTrue "ElasticSearch server is down:
         ${elasticSearchHealth}" '[[ ${elasticSearchHealth} =~ \"status\":\"(green|yellow)\" ]]'
 
-    listOfIndexes="$(vagrant ssh -c 'curl -i http://127.0.0.1:9200/_cat/indices?v')"
+    listOfIndexes="$(executeInMagento2Container curl -- '-i' 'http://elasticsearch-master:9200/_cat/indices?v')"
     assertTrue "Products index must not be available in ElasticSearch:
         ${listOfIndexes}" '[[ ! ${listOfIndexes} =~ magento2_product ]]'
 

init_project.sh

@@ -204,7 +204,7 @@ function refreshSearchIndexes()
     echo "## deleteElasticSearchIndexes" >>${current_log_file_path}
 
     cd "${vagrant_dir}"
-    vagrant ssh -c 'curl -X DELETE -i http://127.0.0.1:9200/_all' >>${current_log_file_path} 2>&1
+    executeInMagento2Container curl -- '-X' 'DELETE' '-i' 'http://elasticsearch-master:9200/_all' >>${current_log_file_path} 2>&1
     bash m-bin-magento indexer:reindex catalogsearch_fulltext >>${current_log_file_path} 2>&1
 }
 

m-search-engine

@@ -11,6 +11,11 @@ source include/configuration.sh
 source include/helpers.sh
 source include/assertions.sh
 
+original_vagrant_dir="${vagrant_dir}"
+source ./../scripts/functions.sh
+vagrant_dir=${original_vagrant_dir}
+cd ${tests_dir}
+
 ## Setup and tear down
 
 function oneTimeSetUp
@@ -58,6 +63,8 @@ function testNoCustomConfigBasicTest()
     assertTestsConfigured
     assertDebugConfigurationWork
     assertRedisCacheIsEnabled
+
+    executeExtendedCommonAssertions
 }
 
 ## Call and Run all Tests