add preprocess option to find_in_batches in ActiveRecord adapter

elasticsearch-model/README.md

@@ -653,6 +653,28 @@ response.records.records.class
 More examples can be found in the `examples` folder. Please see the `Elasticsearch::Model::Adapter`
 module and its submodules for technical information.
 
+#### Customizing `Elasticsearch::Model::Adapter::ActiveRecord`
+
+If you need to integrate with an external service (such as Quickbooks) to get content during an import, you might need to run each batch of records through a preprocessor before importing. Use the `preprocess` option to specify the method used to preprocess each batch:
+
+```ruby
+class PurchaseOrder
+  # Queries the external service in batch
+  # to improve performance and reduce requests.
+  def add_content_from_quickbooks(batch)
+    quickbooks_purchase_orders = Hash[QuickbooksPurchaseOrder.where(id: batch.collect(&:quickbooks_id)).collect { |qpo| [qpo.id, qpo] }]
+    batch.each do |purchase_order|
+      purchase_order.in_quickbooks = quickbooks_purchase_orders[purchase_order.quickbooks_id]
+    end
+    batch
+  end
+end
+
+PurchaseOrder.import(preprocess: :add_content_from_quickbooks)
+
+```
+
+
 ## Development and Community
 
 For local development, clone the repository and run `bundle install`. See `rake -T` for a list of

elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb

@@ -84,11 +84,12 @@ module Importing
           #
           def __find_in_batches(options={}, &block)
             named_scope = options.delete(:scope)
+            preprocess = options.delete(:preprocess)
 
             scope = named_scope ? self.__send__(named_scope) : self
 
             scope.find_in_batches(options) do |batch|
-              yield batch
+              yield (preprocess ? self.__send__(preprocess, batch) : batch)
             end
           end
 

elasticsearch-model/lib/elasticsearch/model/importing.rb

@@ -85,7 +85,7 @@ def import(options={}, &block)
 
           unless transform.respond_to?(:call)
             raise ArgumentError,
-                  "Pass an object responding to `call` as the :transport option, #{transform.class} given"
+                  "Pass an object responding to `call` as the :transform option, #{transform.class} given"
           end
 
           if options.delete(:force)

elasticsearch-model/test/unit/adapter_active_record_test.rb

@@ -16,6 +16,20 @@ def response
       def ids
         [2, 1]
       end
+
+      class << self
+        def augment_collection(batch)
+          # This might update the batch with content from an external system (like Quickbooks).
+          # Performing that external query in batch might be required for performance/throttling reasons.
+
+          # This is just a silly example for the test.
+          batch.collect { |b| b.to_s + '!' }
+        end
+
+        def find_in_batches(options={}, &block)
+          yield [:a, :b]
+        end
+      end
     end
 
     RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [] } }
@@ -104,6 +118,15 @@ def ids
         DummyClassForActiveRecord.__find_in_batches(scope: :published) do; end
       end
 
+      should "preprocess the batch if option provided" do
+        DummyClassForActiveRecord.__find_in_batches(preprocess: :augment_collection) do |batch|
+          batch.each do |b|
+            # the example augment collection method adds an '!' to the end of each element
+            assert_equal b[-1], '!'
+          end
+        end
+      end
+
       context "when transforming models" do
         setup do
           @transform = DummyClassForActiveRecord.__transform