[MODEL] Retouched the code, tests and examples for the preprocess option

Related: #151, #152

Author: karmi

elasticsearch-model/README.md

@@ -146,6 +146,9 @@ Article.import
 # => 0
 ```
 
+It's possible to import only records from a specific `scope`, or transform the batch with the `transform`
+and `preprocess` options -- look for examples in the method documentation.
+
 No errors were reported during importing, so... let's search the index!
 
 
@@ -657,28 +660,6 @@ response.records.records.class
 More examples can be found in the `examples` folder. Please see the `Elasticsearch::Model::Adapter`
 module and its submodules for technical information.
 
-#### Customizing `Elasticsearch::Model::Adapter::ActiveRecord`
-
-If you need to integrate with an external service (such as Quickbooks) to get content during an import, you might need to run each batch of records through a preprocessor before importing. Use the `preprocess` option to specify the method used to preprocess each batch:
-
-```ruby
-class PurchaseOrder
-  # Queries the external service in batch
-  # to improve performance and reduce requests.
-  def add_content_from_quickbooks(batch)
-    quickbooks_purchase_orders = Hash[QuickbooksPurchaseOrder.where(id: batch.collect(&:quickbooks_id)).collect { |qpo| [qpo.id, qpo] }]
-    batch.each do |purchase_order|
-      purchase_order.in_quickbooks = quickbooks_purchase_orders[purchase_order.quickbooks_id]
-    end
-    batch
-  end
-end
-
-PurchaseOrder.import(preprocess: :add_content_from_quickbooks)
-
-```
-
-
 ## Development and Community
 
 For local development, clone the repository and run `bundle install`. See `rake -T` for a list of

elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb

@@ -78,7 +78,8 @@ def self.included(base)
 
         module Importing
 
-          # Fetch batches of records from the database
+          # Fetch batches of records from the database (used by the import method)
+          #
           #
           # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches
           #

elasticsearch-model/lib/elasticsearch/model/importing.rb

@@ -76,6 +76,20 @@ module ClassMethods
         #
         #    Article.import transform: transform
         #
+        # @example Update the batch before yielding it
+        #
+        #     class Article
+        #       # ...
+        #       def enrich(batch)
+        #         batch.each do |item|
+        #           item.metadata = MyAPI.get_metadata(item.id)
+        #         end
+        #         batch
+        #       end
+        #     end
+        #
+        #    Article.import preprocess: enrich
+        #
         def import(options={}, &block)
           errors       = 0
           refresh      = options.delete(:refresh)   || false

elasticsearch-model/test/unit/adapter_active_record_test.rb

@@ -16,20 +16,6 @@ def response
       def ids
         [2, 1]
       end
-
-      class << self
-        def augment_collection(batch)
-          # This might update the batch with content from an external system (like Quickbooks).
-          # Performing that external query in batch might be required for performance/throttling reasons.
-
-          # This is just a silly example for the test.
-          batch.collect { |b| b.to_s + '!' }
-        end
-
-        def find_in_batches(options={}, &block)
-          yield [:a, :b]
-        end
-      end
     end
 
     RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [] } }
@@ -119,12 +105,20 @@ def find_in_batches(options={}, &block)
       end
 
       should "preprocess the batch if option provided" do
-        DummyClassForActiveRecord.__find_in_batches(preprocess: :augment_collection) do |batch|
-          batch.each do |b|
-            # the example augment collection method adds an '!' to the end of each element
-            assert_equal b[-1], '!'
+        class << DummyClassForActiveRecord
+          # Updates/transforms the batch while fetching it from the database
+          # (eg. with information from an external system)
+          #
+          def update_batch(batch)
+            batch.collect { |b| b.to_s + '!' }
           end
         end
+
+        DummyClassForActiveRecord.expects(:__find_in_batches).returns( [:a, :b] )
+
+        DummyClassForActiveRecord.__find_in_batches(preprocess: :update_batch) do |batch|
+          assert_same_elements ["a!", "b!"], batch
+        end
       end
 
       context "when transforming models" do