Improve array matching

Author: thomas4019

README.md

@@ -85,7 +85,7 @@ Cast strings to number when sorting.
 With MongoDB, you can search a document with a subarray of objects that you want to match when any one of the elements in the array matches.
 This tool implements it in SQL using a subquery, so it will likely not be the efficient on large datasets.
 
-To enable subfield matching, you can pass a third parameter which is either an array of field names that will be assumed
+To enable subfield matching, you can pass a third parameter which is either an array of dotted paths that will be assumed
 to potentially be arrays or `true` if you want it to assume any field can be an array.
 
 Example document:
@@ -110,6 +110,9 @@ This then creates a PostgreSQL query like the following:
 OR EXISTS (SELECT * FROM jsonb_array_elements(data->'courses')
            WHERE jsonb_typeof(data->'courses')='array' AND value->>'distance'='5K'))
 ```
+
+Note: nested paths are not yet supported, so passing ['courses', 'courses.distance'] won't support checking both.
+The first matching path is the one that will be used.
     
 ## Supported Features
 * $eq, $gt, $gte, $lt, $lte, $ne

index.js

@@ -15,24 +15,37 @@ var otherOps = {
   $all: true, $in: true, $nin: true, $not: true, $or: true, $and: true, $elemMatch: true, $regex: true, $type: true, $size: true, $exists: true, $mod: true, $text: true
 }
 
-function shouldQueryAsArray(op, arrayPaths) {
+function getMatchingArrayPath(op, arrayPaths) {
   if (arrayPaths === true) {
     // always assume array path if true is passed
     return true
   }
   if (!arrayPaths || !Array.isArray(arrayPaths)) {
     return false
   }
-  return arrayPaths.some(op => op.startsWith(arrayPaths))
+  return arrayPaths.find(path => op.startsWith(path))
 }
 
-function createElementOrArrayQuery(path, op, value, parent) {
-  const subPath = op.split('.')
-  const innerPath = subPath.length > 1 ? ['value', subPath.pop()] : ['value']
+/**
+ * @param path array path current key
+ * @param op current key, might be a dotted path
+ * @param value
+ * @param parent
+ * @param arrayPathStr
+ * @returns {string|string|*}
+ */
+function createElementOrArrayQuery(path, op, value, parent, arrayPathStr) {
+  const arrayPath = arrayPathStr.split('.')
+  const deeperPath = op.split('.').slice(arrayPath.length)
+  const innerPath = ['value', ...deeperPath]
+  const pathToMaybeArray = path.concat(arrayPath)
+
+  // TODO: nested array paths are not yet supported.
   const singleElementQuery = convertOp(path, op, value, parent, [])
-  path = path.concat(subPath)
-  const text = util.pathToText(path, false)
+
+  const text = util.pathToText(pathToMaybeArray, false)
   const safeArray = `jsonb_typeof(${text})='array' AND`
+
   let arrayQuery = ''
   if (typeof value === 'object' && !Array.isArray(value) && value !== null) {
     if (typeof value['$size'] !== 'undefined') {
@@ -75,8 +88,9 @@ function createElementOrArrayQuery(path, op, value, parent) {
  * @param arrayPaths {Array} List of dotted paths that possibly need to be handled as arrays.
  */
 function convertOp(path, op, value, parent, arrayPaths) {
-  if (shouldQueryAsArray(op, arrayPaths)) {
-    return createElementOrArrayQuery(path, op, value, parent)
+  const arrayPath = getMatchingArrayPath(op, arrayPaths)
+  if (arrayPath) {
+    return createElementOrArrayQuery(path, op, value, parent, arrayPath)
   }
   switch(op) {
     case '$not':

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "mongo-query-to-postgres-jsonb",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "Converts MongoDB queries to postgresql queries for jsonb fields.",
   "main": "index.js",
   "directories": {

package.json

@@ -0,0 +1,17 @@
+FROM mongo:3.4
+RUN apt-get update && apt-get -y install wget
+RUN wget -q https://github.com/mongodb/mongo/archive/r4.2.10.tar.gz
+RUN tar -xzf r4.2.10.tar.gz && mv mongo-r4.2.10 mongo
+RUN wget -q -O - https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get install -y nodejs
+WORKDIR /srv
+RUN npm i pgmongo
+RUN rm -r node_modules/mongo-query-to-postgres-jsonb
+
+RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN apt-get update
+RUN apt-get -y install postgresql
+
+COPY . node_modules/mongo-query-to-postgres-jsonb
+COPY scripts/test.sh .

scripts/Dockerfile

@@ -0,0 +1,3 @@
+docker build --tag mongotest -f scripts/Dockerfile ./
+docker run --rm --name mongotest mongotest
+# docker exec -it mongotest /bin/bash

scripts/run_docker_tests.sh

@@ -0,0 +1,5 @@
+pg_ctlcluster 13 main start
+su -c $'psql -c "ALTER USER postgres WITH PASSWORD \'postgres\';"' - postgres
+su -c $'psql -c "CREATE DATABASE test;"' - postgres
+DEBUG=pgmongo:* node node_modules/.bin/pgmongo localhost &
+mongo --port 27018 ../mongo/jstests/core/where1.js

scripts/test.sh

@@ -190,13 +190,24 @@ describe('$mod', function () {
 
 describe('Match a Field Without Specifying Array Index', function () {
   it('basic case', function() {
-    assert.equal('(data @> \'{ "courses": { "distance": "5K" } }\' OR EXISTS (SELECT * FROM jsonb_array_elements(data->\'courses\') WHERE jsonb_typeof(data->\'courses\')=\'array\' AND value @> \'{ "distance": "5K" }\'))', convert('data', { 'courses.distance': '5K' }, ['courses']))
+    assert.equal('(data @> \'{ "courses": { "distance": "5K" } }\' OR EXISTS (SELECT * FROM jsonb_array_elements' +
+        '(data->\'courses\') WHERE jsonb_typeof(data->\'courses\')=\'array\' AND value @> \'{ "distance": "5K" }\'))',
+    convert('data', { 'courses.distance': '5K' }, ['courses', 'other']))
+  })
+  it('basic deep case', function() {
+    assert.equal('(data @> \'{ "courses": { "distance": "5K" } }\' OR EXISTS (SELECT * FROM jsonb_array_elements' +
+        '(data->\'courses\'->\'distance\') WHERE jsonb_typeof(data->\'courses\'->\'distance\')=\'array\' AND value @> \'"5K"\'))',
+    convert('data', { 'courses.distance': '5K' }, ['courses.distance']))
   })
   it('direct match', function() {
-    assert.equal('(data @> \'{ "roles": "Admin" }\' OR EXISTS (SELECT * FROM jsonb_array_elements(data->\'roles\') WHERE jsonb_typeof(data->\'roles\')=\'array\' AND value @> \'"Admin"\'))', convert('data', { 'roles': 'Admin' }, ['roles']))
+    assert.equal('(data @> \'{ "roles": "Admin" }\' OR EXISTS (SELECT * FROM jsonb_array_elements(' +
+        'data->\'roles\') WHERE jsonb_typeof(data->\'roles\')=\'array\' AND value @> \'"Admin"\'))',
+    convert('data', { 'roles': 'Admin' }, ['roles']))
   })
   it('$in', function() {
-    assert.equal('(data->>\'roles\' IN (\'Test\', \'Admin\') OR EXISTS (SELECT * FROM jsonb_array_elements(data->\'roles\') WHERE jsonb_typeof(data->\'roles\')=\'array\' AND value #>>\'{}\' IN (\'Test\', \'Admin\')))', convert('data', { 'roles': { $in: ['Test', 'Admin'] } }, ['roles']))
+    assert.equal('(data->>\'roles\' IN (\'Test\', \'Admin\') OR EXISTS (SELECT * FROM jsonb_array_elements(' +
+        'data->\'roles\') WHERE jsonb_typeof(data->\'roles\')=\'array\' AND value #>>\'{}\' IN (\'Test\', \'Admin\')))',
+    convert('data', { 'roles': { $in: ['Test', 'Admin'] } }, ['roles']))
   })
 })
 describe('special cases', function () {