7.0 support (elastic#875)

* [MODEL] Updates for 7.0

* [STORE] Updates for 7.0

* [MODEL] Test against Rails 6.0.beta3

* Delete all gemfile.lock files in bundle clean task

* [STORE] Account for no types being accepted for mappings in 7.0

* Update travis config

* Update script to download and start elasticsearch

* Use wait_for_green script before running tests

* Handle downloading and installing ES 6.7.1 nad 7.0.0 on Travis

* Update travis script

* Leave testing against ES 6.x to the 6.x branch

* [MODEL] Add test for handling different total hits format in search response

* Correct merge conflicts

* Update dependency versions for 7.0 release of client

Author: estolfo

Diff for: .travis.yml

@@ -14,10 +14,7 @@ services:
 branches:
   only:
     - master
-    - travis
-    - 5.x
     - 6.x
-    - 2.x
 
 matrix:
   include:
@@ -29,7 +26,7 @@ matrix:
       jdk: oraclejdk8
       env: RAILS_VERSIONS=5.0
 
-    - rvm: 2.6.1
+    - rvm: 2.6.2
       jdk: oraclejdk8
       env: RAILS_VERSIONS=4.0,5.0,6.0
 
@@ -39,12 +36,13 @@ matrix:
 
 env:
   global:
-    - ELASTICSEARCH_VERSION=7.0.0-beta1
+    - ELASTICSEARCH_VERSION=7.0.0
+    - TEST_ES_SERVER=http://localhost:9250
     - TEST_CLUSTER_PORT=9250
     - QUIET=true
 
 before_install:
-  - TEST_CLUSTER_PORT=9250 source ./travis_before_script.sh
+  - source ./travis_before_script.sh
   - gem update --system
   - gem update bundler
   - gem --version

Diff for: Rakefile

@@ -22,6 +22,33 @@ subprojects << 'elasticsearch-model' unless defined?(JRUBY_VERSION)
 
 __current__ = Pathname( File.expand_path('..', __FILE__) )
 
+def admin_client
+  $admin_client ||= begin
+    transport_options = {}
+    test_suite = ENV['TEST_SUITE'].freeze
+
+    if hosts = ENV['TEST_ES_SERVER'] || ENV['ELASTICSEARCH_HOSTS']
+      split_hosts = hosts.split(',').map do |host|
+        /(http\:\/\/)?(\S+)/.match(host)[2]
+      end
+
+      host, port = split_hosts.first.split(':')
+    end
+
+    if test_suite == 'security'
+      transport_options.merge!(:ssl => { verify: false,
+                                         ca_path: CERT_DIR })
+
+      password = ENV['ELASTIC_PASSWORD']
+      user = ENV['ELASTIC_USER'] || 'elastic'
+      url = "https://#{user}:#{password}@#{host}:#{port}"
+    else
+      url = "http://#{host || 'localhost'}:#{port || 9200}"
+    end
+    Elasticsearch::Client.new(host: url, transport_options: transport_options)
+  end
+end
+
 task :default do
   system "rake --tasks"
 end
@@ -53,9 +80,7 @@ namespace :bundle do
     subprojects.each do |project|
       sh "rm -f #{__current__.join(project)}/Gemfile.lock"
     end
-    sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/3.0.gemfile.lock"
-    sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/4.0.gemfile.lock"
-    sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/5.0.gemfile.lock"
+    sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/*.lock"
   end
 end
 
@@ -132,7 +157,7 @@ namespace :test do
   end
 
   desc "Run all tests in all subprojects"
-  task :all do
+  task :all => :wait_for_green do
     subprojects.each do |project|
       puts '-'*80
       sh "cd #{project} && " +
@@ -163,6 +188,32 @@ namespace :test do
   end
 end
 
+
+desc "Wait for elasticsearch cluster to be in green state"
+task :wait_for_green do
+  require 'elasticsearch'
+
+  ready = nil
+  5.times do |i|
+    begin
+      puts "Attempting to wait for green status: #{i + 1}"
+      if admin_client.cluster.health(wait_for_status: 'green', timeout: '50s')
+        ready = true
+        break
+      end
+    rescue Elasticsearch::Transport::Transport::Errors::RequestTimeout => ex
+      puts "Couldn't confirm green status.\n#{ex.inspect}."
+    rescue Faraday::ConnectionFailed => ex
+      puts "Couldn't connect to Elasticsearch.\n#{ex.inspect}."
+      sleep(30)
+    end
+  end
+  unless ready
+    puts "Couldn't connect to Elasticsearch, aborting program."
+    exit(1)
+  end
+end
+
 desc "Generate documentation for all subprojects"
 task :doc do
   subprojects.each do |project|

Diff for: elasticsearch-model/.gitignore

@@ -16,4 +16,6 @@ test/tmp
 test/version_tmp
 tmp
 
-gemfiles/*.gemfile.lock
+
+gemfiles/*.lock
+

Diff for: elasticsearch-model/gemfiles/6.0.gemfile

@@ -20,6 +20,7 @@
 # $ BUNDLE_GEMFILE=./gemfiles/6.0.gemfile bundle install
 # $ BUNDLE_GEMFILE=./gemfiles/6.0.gemfile bundle exec rake test:integration
 
+
 source 'https://rubygems.org'
 
 gemspec path: '../'

Diff for: elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb

@@ -75,8 +75,8 @@ def __records_for_klass(klass, ids)
                 klass.where(klass.primary_key => ids)
               when Elasticsearch::Model::Adapter::Mongoid.equal?(adapter)
                 klass.where(:id.in => ids)
-              else
-                klass.find(ids)
+            else
+              klass.find(ids)
             end
           end
 
@@ -108,13 +108,21 @@ def __ids_by_type
           def __type_for_hit(hit)
             @@__types ||= {}
 
-            @@__types[ "#{hit[:_index]}::#{hit[:_type]}" ] ||= begin
+            key = "#{hit[:_index]}::#{hit[:_type]}" if hit[:_type] && hit[:_type] != '_doc'
+            key = hit[:_index] unless key
+
+            @@__types[key] ||= begin
               Registry.all.detect do |model|
-                model.index_name == hit[:_index] && model.document_type == hit[:_type]
+                (model.index_name == hit[:_index] && __no_type?(hit)) ||
+                    (model.index_name == hit[:_index] && model.document_type == hit[:_type])
               end
             end
           end
 
+          def __no_type?(hit)
+            hit[:_type].nil? || hit[:_type] == '_doc'
+          end
+
           # Returns the adapter registered for a particular `klass` or `nil` if not available
           #
           # @api private

Diff for: elasticsearch-model/lib/elasticsearch/model/indexing.rb

@@ -56,9 +56,7 @@ class Mappings
         # @private
         TYPES_WITH_EMBEDDED_PROPERTIES = %w(object nested)
 
-        def initialize(type, options={})
-          raise ArgumentError, "`type` is missing" if type.nil?
-
+        def initialize(type = nil, options={})
           @type    = type
           @options = options
           @mapping = {}
@@ -89,7 +87,11 @@ def indexes(name, options={}, &block)
         end
 
         def to_hash
-          { @type.to_sym => @options.merge( properties: @mapping ) }
+          if @type
+            { @type.to_sym => @options.merge( properties: @mapping ) }
+          else
+            @options.merge( properties: @mapping )
+          end
         end
 
         def as_json(options={})
@@ -246,10 +248,12 @@ def create_index!(options={})
           delete_index!(options.merge index: target_index) if options[:force]
 
           unless index_exists?(index: target_index)
-            self.client.indices.create index: target_index,
-                                       body: {
-                                         settings: settings,
-                                         mappings: mappings }
+            options.delete(:force)
+            self.client.indices.create({ index: target_index,
+                                         body: {
+                                           settings: settings,
+                                           mappings: mappings }
+                                       }.merge(options))
           end
         end
 

Diff for: elasticsearch-model/lib/elasticsearch/model/naming.rb

@@ -108,10 +108,7 @@ def default_index_name
             self.model_name.collection.gsub(/\//, '-')
           end
 
-          def default_document_type
-            DEFAULT_DOC_TYPE
-          end
-
+          def default_document_type; end
       end
 
       module InstanceMethods