Skip to content

Commit d4deea6

Browse files
author
Thomas Hansen
committed
Improve array queries and other minor changes.
* Adds option for sorting by numeric strings * Adds support for upsert queries with $inc, $push, etc. * Style fixes
1 parent 65cfaf8 commit d4deea6

9 files changed

+110
-59
lines changed

‎.eslintrc.js

+3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ module.exports = {
44
"mocha": true
55
},
66
"extends": "eslint:recommended",
7+
"parserOptions": {
8+
"ecmaVersion": 2017,
9+
},
710
"rules": {
811
"indent": [
912
"error",

‎index.js

+24-17
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
var util = require('./util.js')
22

33
// These are the simple operators.
4+
// Note that "is distinct from" needs to be used to ensure nulls are returned as expected, see https://modern-sql.com/feature/is-distinct-from
45
var ops = {
56
$eq: '=',
67
$gt: '>',
78
$gte: '>=',
89
$lt: '<',
910
$lte: '<=',
10-
$ne: '!=',
11+
$ne: ' IS DISTINCT FROM ',
1112
}
1213

1314
var otherOps = {
@@ -23,8 +24,8 @@ function convertOp(path, op, value, parent, arrayPaths) {
2324
const singleElementQuery = convertOp(path, op, value, parent, [])
2425
path = path.concat(subPath)
2526
const text = util.pathToText(path, false)
26-
const safeArray = "jsonb_typeof(data->'a')='array' AND";
27-
let arrayQuery = '';
27+
const safeArray = `jsonb_typeof(${text})='array' AND`
28+
let arrayQuery = ''
2829
if (typeof value === 'object' && !Array.isArray(value) && value !== null) {
2930
if (value['$elemMatch']) {
3031
const sub = convert(innerPath, value['$elemMatch'], [], false)
@@ -60,8 +61,8 @@ function convertOp(path, op, value, parent, arrayPaths) {
6061
case '$not':
6162
return '(NOT ' + convert(path, value) + ')'
6263
case '$nor':
63-
var notted = value.map((e) => ({ $not: e }));
64-
return convertOp(path, '$and', notted, value, arrayPaths);
64+
var notted = value.map((e) => ({ $not: e }))
65+
return convertOp(path, '$and', notted, value, arrayPaths)
6566
case '$or':
6667
case '$and':
6768
if (!Array.isArray(value)) {
@@ -71,7 +72,7 @@ function convertOp(path, op, value, parent, arrayPaths) {
7172
throw new Error('$and/$or/$nor must be a nonempty array')
7273
} else {
7374
for (const v of value) {
74-
if (typeof v !== "object") {
75+
if (typeof v !== 'object') {
7576
throw new Error('$or/$and/$nor entries need to be full objects')
7677
}
7778
}
@@ -94,7 +95,7 @@ function convertOp(path, op, value, parent, arrayPaths) {
9495
return partial
9596
case '$regex':
9697
var op = '~'
97-
var op2 = '';
98+
var op2 = ''
9899
if (parent['$options'] && parent['$options'].includes('i')) {
99100
op += '*'
100101
}
@@ -117,15 +118,20 @@ function convertOp(path, op, value, parent, arrayPaths) {
117118
var text = util.pathToText(path, false)
118119
return 'jsonb_array_length(' + text + ')=' + value
119120
case '$exists':
120-
const key = path.pop();
121-
var text = util.pathToText(path, false)
122-
return text + ' ? ' + util.quote(key)
121+
if (path.length > 1) {
122+
const key = path.pop()
123+
var text = util.pathToText(path, false)
124+
return (value ? '' : ' NOT ') + text + ' ? ' + util.quote(key)
125+
} else {
126+
var text = util.pathToText(path, false)
127+
return text + ' IS ' + (value ? 'NOT ' : '') + 'NULL'
128+
}
123129
case '$mod':
124130
var text = util.pathToText(path, true)
125131
if (typeof value[0] != 'number' || typeof value[1] != 'number') {
126132
throw new Error('$mod requires numeric inputs')
127133
}
128-
return 'cast(' + text + ' AS numeric) % ' + value[0] + '=' + value[1];
134+
return 'cast(' + text + ' AS numeric) % ' + value[0] + '=' + value[1]
129135
default:
130136
return convert(path.concat(op.split('.')), value)
131137
}
@@ -165,11 +171,11 @@ var convert = function (path, query, arrayPaths, forceExact=false) {
165171
var text = util.pathToText(path, typeof query == 'string')
166172
return text + '=' + util.quote(query)
167173
case 1:
168-
const key = specialKeys[0];
169-
return convertOp(path, key, query[key], query, arrayPaths);
174+
const key = specialKeys[0]
175+
return convertOp(path, key, query[key], query, arrayPaths)
170176
default:
171177
return '(' + specialKeys.map(function (key) {
172-
return convertOp(path, key, query[key], query, arrayPaths);
178+
return convertOp(path, key, query[key], query, arrayPaths)
173179
}).join(' and ') + ')'
174180
}
175181
}
@@ -180,6 +186,7 @@ module.exports = function (fieldName, query, arrays) {
180186
}
181187
module.exports.convertDotNotation = util.convertDotNotation
182188
module.exports.pathToText = util.pathToText
183-
module.exports.convertSelect = require('./select');
184-
module.exports.convertUpdate = require('./update');
185-
module.exports.convertSort = require('./sort');
189+
module.exports.countUpdateSpecialKeys = util.countUpdateSpecialKeys
190+
module.exports.convertSelect = require('./select')
191+
module.exports.convertUpdate = require('./update')
192+
module.exports.convertSort = require('./sort')

‎select.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ var convert = function (fieldName, projection) {
77
}
88
//var output = '';
99
var shellDoc = {}
10-
var removals = [];
10+
var removals = []
1111
Object.keys(projection).forEach(function(field) {
1212
var path = field.split('.')
1313
if (projection[field] === 1) {
@@ -52,7 +52,7 @@ var convert = function (fieldName, projection) {
5252
return 'jsonb_build_object(' + entries.join(', ') + ')'
5353
}
5454
}
55-
var out = Object.keys(shellDoc).length > 0 ? convertRecur(shellDoc) : fieldName;
55+
var out = Object.keys(shellDoc).length > 0 ? convertRecur(shellDoc) : fieldName
5656
if (removals.length) {
5757
out += ' ' + removals.join(' ')
5858
}

‎sort.js

+10-7
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
var util = require('./util.js')
22

3-
var convertField = function (fieldName, field, orderingType) {
3+
var convertField = function (fieldName, field, orderingType, forceNumericSort) {
44
const dir = (orderingType === 1 ? 'ASC NULLS FIRST' : 'DESC NULLS LAST')
5-
const value = util.pathToText([fieldName].concat(field.split('.')), false)
6-
return value + ' ' + dir;
5+
const value = util.pathToText([fieldName].concat(field.split('.')), forceNumericSort)
6+
if (forceNumericSort) {
7+
return `cast(${value} as double precision) ${dir}`
8+
}
9+
return `${value} ${dir}`
710
}
811

9-
var convert = function (fieldName, sortParams) {
12+
var convert = function (fieldName, sortParams, forceNumericSort) {
1013
const orderings = Object.keys(sortParams).map(function(key) {
11-
return convertField(fieldName, key, sortParams[key])
12-
});
13-
return orderings.join(', ');
14+
return convertField(fieldName, key, sortParams[key], forceNumericSort || false)
15+
})
16+
return orderings.join(', ')
1417
}
1518

1619
module.exports = convert

‎test/filter.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ describe('array equality', function () {
2424
assert.equal('data->\'roles\'->>0=\'Admin\'', convert('data', {'roles.0': 'Admin'}))
2525
})
2626
it('support element matching', function() {
27-
assert.equal('data->\'roles\' @> \'"Admin"\'::jsonb', convert('data', {'roles': {$elemMatch: 'Admin'}}))
27+
assert.equal('data->>\'roles\'=\'Admin\'', convert('data', {'roles': {$elemMatch: 'Admin'}}))
2828
})
2929
})
3030

@@ -104,7 +104,7 @@ describe('comparision operators', function() {
104104
assert.equal('data->>\'type\'=\'food\'', convert('data', { type: { $eq : 'food' } }))
105105
})
106106
it('$ne', function () {
107-
assert.equal('data->>\'type\'!=\'food\'', convert('data', { type: { $ne : 'food' } }))
107+
assert.equal('data->>\'type\' IS DISTINCT FROM \'food\'', convert('data', { type: { $ne : 'food' } }))
108108
})
109109
it('$gt', function () {
110110
assert.equal('data->\'count\'>\'5\'::jsonb', convert('data', { count: { $gt : 5 } }))

‎test/select.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ describe('select: ', function() {
3030

3131
describe('excluded fields', function () {
3232
it('single excluded', function () {
33-
assert.equal(convertSelect('data', { b: 0 }), "data #- '{b}' as data")
33+
assert.equal(convertSelect('data', { b: 0 }), 'data #- \'{b}\' as data')
3434
})
3535
it('exclude deep', function () {
36-
assert.equal(convertSelect('data', { 'field.inner': 0 }), "data #- '{field,inner}' as data")
36+
assert.equal(convertSelect('data', { 'field.inner': 0 }), 'data #- \'{field,inner}\' as data')
3737
})
3838
it('combined exclusion and inclusion', function () {
3939
assert.throws(() => convertSelect('data', { a: 1, b: 0 }), 'Projection cannot have a mix of inclusion and exclusion.')

‎test/update.js

+7-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ describe('update: ', function() {
2020
it('$set multiple', function() {
2121
assert.equal(convertUpdate('data', { $set: { field: 'value', second: 2 } }), 'jsonb_set(jsonb_set(data,\'{second}\',\'2\'::jsonb),\'{field}\',\'"value"\')')
2222
})
23+
it('$set deep', function() {
24+
assert.equal(convertUpdate('data', { $set: { 'a.b': 2 } }), 'jsonb_set(jsonb_set(data,\'{a}\',COALESCE(data->\'a\', \'{}\'::jsonb)),\'{a,b}\',\'2\'::jsonb)')
25+
})
2326

2427
it('$unset', function() {
2528
assert.equal(convertUpdate('data', { $unset: { field: 'value' } }), 'data #- \'{field}\'')
@@ -53,7 +56,10 @@ describe('update: ', function() {
5356
assert.equal(convertUpdate('data', { $pull: { cities: 'LA' } }), 'array_remove(ARRAY(SELECT value FROM jsonb_array_elements(data->\'cities\')),\'"LA"\')')
5457
})
5558
it('$push', function() {
56-
assert.equal(convertUpdate('data', { $push: { cities: 'LA' } }), 'jsonb_set(data,\'{cities}\',to_jsonb(array_append(ARRAY(SELECT value FROM jsonb_array_elements(data->\'cities\') WHERE value != \'"LA"\'),\'"LA"\')))')
59+
assert.equal(convertUpdate('data', { $push: { cities: 'LA' } }), 'jsonb_set(data,\'{cities}\',to_jsonb(array_append(ARRAY(SELECT value FROM jsonb_array_elements(data->\'cities\')),\'"LA"\')))')
60+
})
61+
it('$addToSet', function() {
62+
assert.equal(convertUpdate('data', { $addToSet: { cities: 'LA' } }), 'jsonb_set(data,\'{cities}\',to_jsonb(array_append(ARRAY(SELECT value FROM jsonb_array_elements(data->\'cities\') WHERE value != \'"LA"\'),\'"LA"\')))')
5763
})
5864
})
5965

‎update.js

+51-27
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,84 @@
1-
var util = require('./util.js')
1+
const _ = require('lodash')
2+
const util = require('./util.js')
23

3-
const specialKeys = ['$currentDate', '$inc', '$min', '$max', '$mul', '$rename', '$set', '$setOnInsert', '$unset', '$push', '$pull']
4-
5-
function convertOp(input, op, data, fieldName) {
6-
const pathText = Object.keys(data)[0];
7-
const value = data[pathText];
8-
delete data[pathText];
4+
function convertOp(input, op, data, fieldName, upsert) {
5+
const pathText = Object.keys(data)[0]
6+
const value = data[pathText]
7+
delete data[pathText]
98
if (Object.keys(data).length > 0) {
10-
input = convertOp(input, op, data, fieldName)
9+
input = convertOp(input, op, data, fieldName, upsert)
1110
}
12-
const path = pathText.split('.');
13-
const pgPath = util.toPostgresPath(path);
11+
const path = pathText.split('.')
12+
const pgPath = util.toPostgresPath(path)
1413
const pgQueryPath = util.pathToText([fieldName].concat(path), false)
1514
const pgQueryPathStr = util.pathToText([fieldName].concat(path), true)
15+
const prevNumericVal = upsert ? '0' : util.toNumeric(pgQueryPathStr)
1616
switch (op) {
1717
case '$set':
18-
if (path.pop() === '_id') {
18+
// Create the necessary top level keys since jsonb_set will not create them automatically.
19+
if (path.length > 1) {
20+
for (let i = 0; i < path.length - 1; i++) {
21+
const parentPath = util.toPostgresPath([path[i]])
22+
if (!input.includes(parentPath)) {
23+
const parentValue = upsert ? '\'{}\'::jsonb' : `COALESCE(${util.pathToText([fieldName].concat(path.slice(0, i + 1)))}, '{}'::jsonb)`
24+
input = 'jsonb_set(' + input + ',' + parentPath + ',' + parentValue + ')'
25+
}
26+
}
27+
}
28+
if (_.last(path) === '_id' && !upsert) {
1929
throw new Error('Mod on _id not allowed')
2030
}
2131
return 'jsonb_set(' + input + ',' + pgPath + ',' + util.quote2(value) + ')'
22-
break;
2332
case '$unset':
24-
return input + ' #- ' + pgPath;
33+
return input + ' #- ' + pgPath
2534
case '$inc':
26-
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(' + util.toNumeric(pgQueryPathStr) + '+' + value + '))'
35+
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(' + prevNumericVal + '+' + value + '))'
2736
case '$mul':
28-
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(' + util.toNumeric(pgQueryPathStr) + '*' + value + '))'
37+
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(' + prevNumericVal + '*' + value + '))'
2938
case '$min':
30-
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(LEAST(' + util.toNumeric(pgQueryPathStr) + ',' + value + ')))'
39+
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(LEAST(' + prevNumericVal + ',' + value + ')))'
3140
case '$max':
32-
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(GREATEST(' + util.toNumeric(pgQueryPathStr) + ',' + value + ')))'
41+
return 'jsonb_set(' + input + ',' + pgPath + ',to_jsonb(GREATEST(' + prevNumericVal + ',' + value + ')))'
3342
case '$rename':
3443
const pgNewPath = util.toPostgresPath(value.split('.'))
35-
return 'jsonb_set(' + input + ',' + pgNewPath + ',' + pgQueryPath + ') #- ' + pgPath;
44+
return 'jsonb_set(' + input + ',' + pgNewPath + ',' + pgQueryPath + ') #- ' + pgPath
3645
case '$pull':
3746
return 'array_remove(ARRAY(SELECT value FROM jsonb_array_elements(' + pgQueryPath + ')),' + util.quote2(value) + ')'
3847
case '$push':
39-
const v = util.quote2(value);
48+
const v2 = util.quote2(value)
49+
if (upsert) {
50+
const newArray = 'jsonb_build_array(' + v2 + ')'
51+
return 'jsonb_set(' + input + ',' + pgPath + ',' + newArray + ')'
52+
}
53+
const updatedArray2 = 'to_jsonb(array_append(ARRAY(SELECT value FROM jsonb_array_elements(' + pgQueryPath + ')),' + v2 + '))'
54+
return 'jsonb_set(' + input + ',' + pgPath + ',' + updatedArray2 + ')'
55+
case '$addToSet':
56+
const v = util.quote2(value)
57+
if (upsert) {
58+
const newArray = 'jsonb_build_array(' + v + ')'
59+
return 'jsonb_set(' + input + ',' + pgPath + ',' + newArray + ')'
60+
}
4061
const updatedArray = 'to_jsonb(array_append(ARRAY(SELECT value FROM jsonb_array_elements(' + pgQueryPath + ') WHERE value != ' + v + '),' + v + '))'
4162
return 'jsonb_set(' + input + ',' + pgPath + ',' + updatedArray + ')'
4263
}
4364
}
4465

45-
var convert = function (fieldName, update) {
46-
var specialCount = Object.keys(update).filter(function(n) {
47-
return specialKeys.includes(n)
48-
}).length;
66+
var convert = function (fieldName, update, upsert) {
67+
var specialCount = util.countUpdateSpecialKeys(update)
4968
if (specialCount === 0) {
5069
return '\'' + JSON.stringify(update) + '\'::jsonb'
5170
}
52-
var output = fieldName
53-
Object.keys(update).forEach(function(key) {
54-
if (!specialKeys.includes(key)) {
71+
var output = upsert ? '\'{}\'::jsonb' : fieldName
72+
let keys = Object.keys(update)
73+
// $set needs to happen first
74+
if (keys.includes('$set')) {
75+
keys = ['$set'].concat(_.pull(keys, '$set'))
76+
}
77+
keys.forEach(function(key) {
78+
if (!util.updateSpecialKeys.includes(key)) {
5579
throw new Error('The <update> document must contain only update operator expressions.')
5680
}
57-
output = convertOp(output, key, update[key], fieldName)
81+
output = convertOp(output, key, _.cloneDeep(update[key]), fieldName, upsert)
5882
})
5983
return output
6084
}

‎util.js

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
exports.updateSpecialKeys = ['$currentDate', '$inc', '$min', '$max', '$mul', '$rename', '$set', '$setOnInsert', '$unset', '$push', '$pull', '$addToSet']
2+
3+
exports.countUpdateSpecialKeys = function(doc) {
4+
return Object.keys(doc).filter(function(n) {
5+
return exports.updateSpecialKeys.includes(n)
6+
}).length
7+
}
8+
19
exports.quote = function(data) {
210
if (typeof data == 'string')
311
return '\'' + exports.stringEscape(data) + '\''
@@ -21,7 +29,7 @@ exports.pathToText = function(path, isString) {
2129
}
2230
for (var i = 1; i < path.length; i++) {
2331
text += (i == path.length-1 && isString ? '->>' : '->')
24-
if (/\d+/.test(path[i]))
32+
if (/^\d+$/.test(path[i]))
2533
text += path[i] //don't wrap numbers in quotes
2634
else
2735
text += '\'' + exports.stringEscape(path[i]) + '\''

0 commit comments

Comments
 (0)