[MODEL] Added a Article.import method for easy and efficient importing records from the database

* Supports both including into class directly and into the proxy (default)
* Uses `find_in_batches` in Activerecord
* Uses cursor (`all.each`) in Mongoid
* Supports other stores via pluggable strategy

Author: karmi

elasticsearch-model/lib/elasticsearch/model.rb

@@ -13,6 +13,7 @@
 require 'elasticsearch/model/adapters/active_record'
 require 'elasticsearch/model/adapters/mongoid'
 
+require 'elasticsearch/model/importing'
 require 'elasticsearch/model/indexing'
 require 'elasticsearch/model/naming'
 require 'elasticsearch/model/serializing'
@@ -80,10 +81,18 @@ def self.included(base)
           include Elasticsearch::Model::Serializing::InstanceMethods
         end
 
+        # Delegate important methods to the `__elasticsearch__` proxy, unless they are defined already
+        #
         extend  Support::Forwardable
         forward :'self.__elasticsearch__', :search   unless respond_to?(:search)
         forward :'self.__elasticsearch__', :mapping  unless respond_to?(:mapping)
         forward :'self.__elasticsearch__', :settings unless respond_to?(:settings)
+        forward :'self.__elasticsearch__', :import   unless respond_to?(:import)
+
+        # Mix the importing module into the proxy
+        #
+        self.__elasticsearch__.class.__send__ :include, Elasticsearch::Model::Importing::ClassMethods
+        self.__elasticsearch__.class.__send__ :include, Adapter.from_class(base).importing_mixin
       end
     end
 

elasticsearch-model/lib/elasticsearch/model/adapter.rb

@@ -28,6 +28,10 @@ def callbacks_mixin
           adapter.const_get(:Callbacks)
         end
 
+        def importing_mixin
+          adapter.const_get(:Importing)
+        end
+
         def adapter
           @adapter ||= begin
             self.class.adapters.find( lambda {[]} ) { |name, condition| condition.call(klass) }.first \

elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb

@@ -57,6 +57,19 @@ def self.included(base)
           end
         end
 
+        module Importing
+          # Fetch batches of records from the database
+          #
+          # Use the [`find_in_batches`](http://api.rubyonrails.org/classes/ActiveRecord/Batches.html) method
+          #
+          def __find_in_batches(options={}, &block)
+            find_in_batches(options) do |batch|
+              batch_for_bulk = batch.as_json.map { |a| { index: { _id: a.delete('id'), data: a } } }
+              yield batch_for_bulk
+            end
+          end
+        end
+
       end
 
     end

elasticsearch-model/lib/elasticsearch/model/adapters/default.rb

@@ -4,7 +4,6 @@ module Adapter
       module Default
 
         module Records
-
           # Use `ActiveModel#find`
           #
           def records
@@ -16,6 +15,12 @@ module Callbacks
           # noop
         end
 
+        module Importing
+          def __find_in_batches(options={}, &block)
+            raise NoMethodError, "Method not implemented for default adapter"
+          end
+        end
+
       end
     end
   end

elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb

@@ -43,6 +43,33 @@ def self.included(base)
           end
         end
 
+        module Importing
+          # Fetch batches of records from the database
+          #
+          # @see https://github.com/mongoid/mongoid/issues/1334
+          # @see https://github.com/karmi/retire/pull/724
+          #
+          def __find_in_batches(options={}, &block)
+            options[:batch_size] ||= 1_000
+            items = []
+
+            all.each do |item|
+              items << item
+
+              if items.length % options[:batch_size] == 0
+                batch_for_bulk = items.map { |a| { index: { _id: a.id, data: a.attributes } } }
+                yield batch_for_bulk
+                items = []
+              end
+            end
+
+            unless items.empty?
+              batch_for_bulk = items.map { |a| { index: { _id: a.id, data: a.attributes } } }
+              yield batch_for_bulk
+            end
+          end
+        end
+
       end
 
     end

elasticsearch-model/lib/elasticsearch/model/importing.rb

@@ -0,0 +1,51 @@
+module Elasticsearch
+  module Model
+
+    # This module provides the support for easily and efficiently importing
+    #  all the records from the including class into the index.
+    #
+    module Importing
+
+      module ClassMethods
+
+        def self.included(base)
+          adapter = Adapter.from_class(base)
+          base.__send__ :include, adapter.importing_mixin
+        end
+
+        # Import all model records into the index
+        #
+        # The method will pick up correct strategy based on the `Importing` module
+        # defined in the corresponding adapter.
+        #
+        # @example Import all records into the index
+        #
+        #     Article.import
+        #
+        # @example Set the batch size to 100
+        #
+        #     Article.import(batch_size: 100)
+        #
+        # @example Process the response from Elasticsearch
+        #
+        #     Article.import do |response|
+        #       puts "Got " + response['items'].select { |i| i['index']['error'] }.size.to_s + " errors"
+        #     end
+        #
+        def import(options={}, &block)
+          __find_in_batches(options) do |batch|
+            response = client.bulk \
+                         index: index_name,
+                         type:  document_type,
+                         body:  batch,
+                         refresh: options[:refresh]
+            yield response if block_given?
+          end
+        end
+
+      end
+
+    end
+
+  end
+end