perf: improve paging query performance

benjamin658 · benjamin658 · commit ff6e10d75488 · 2020-07-02T19:21:24.000+08:00
- use inner join on sub query paging table to prevent TypeORM executes id IN (...) query
diff --git a/src/Paginator.ts b/src/Paginator.ts
@@ -113,35 +113,50 @@ export default class Paginator<Entity> {
       Object.assign(cursors, this.decode(this.beforeCursor as string));
     }
 
-    if (Object.keys(cursors).length > 0) {
-      builder.addFrom((pagingQuery) => pagingQuery
+    builder.innerJoin((paging) => {
+      const pagingSubQuery = paging
+        .from(this.entity, 'paging')
         .select(this.paginationKeys)
-        .from(this.entity, this.alias)
-        .andWhere(new Brackets((where) => this.buildCursor(where, cursors))),
-      'paging');
+        .limit(this.limit + 1)
+        .orderBy(this.buildOrder('paging'));
 
-      this.paginationKeys.forEach((key) => {
-        builder.andWhere(`${this.alias}.${key} = paging.${key}`);
-      });
-    }
+      if (Object.keys(cursors).length > 0) {
+        pagingSubQuery
+          .andWhere(new Brackets((where) => this.buildCursorQuery(where, cursors)));
+      }
+
+      return pagingSubQuery;
+    },
+    'paging',
+    this.buildPagingInnerJoinCondition());
 
-    builder.take(this.limit + 1);
-    builder.orderBy(this.buildOrder());
+    builder.orderBy(this.buildOrder(this.alias));
 
     return builder;
   }
 
-  private buildCursor(where: WhereExpression, cursors: CursorParam): void {
+  private buildCursorQuery(where: WhereExpression, cursors: CursorParam): void {
     const operator = this.getOperator();
     const params: CursorParam = {};
     let query = '';
     this.paginationKeys.forEach((key) => {
       params[key] = cursors[key];
-      where.orWhere(`${query}${this.alias}.${key} ${operator} :${key}`, params);
-      query = `${query}${this.alias}.${key} = :${key} AND `;
+      where.orWhere(`${query}paging.${key} ${operator} :${key}`, params);
+      query = `${query}paging.${key} = :${key} AND `;
     });
   }
 
+  private buildPagingInnerJoinCondition(): string {
+    return this.paginationKeys.reduce((prev, next) => {
+      let query = `${this.alias}.${next} = paging.${next}`;
+      if (prev !== '') {
+        query = `AND ${query}`;
+      }
+
+      return `${prev} ${query}`;
+    }, '');
+  }
+
   private getOperator(): string {
     if (this.hasAfterCursor()) {
       return this.order === 'ASC' ? '>' : '<';
@@ -154,7 +169,7 @@ export default class Paginator<Entity> {
     return '=';
   }
 
-  private buildOrder(): OrderByCondition {
+  private buildOrder(alias: string): OrderByCondition {
     let { order } = this;
 
     if (!this.hasAfterCursor() && this.hasBeforeCursor()) {
@@ -163,7 +178,7 @@ export default class Paginator<Entity> {
 
     const orderByCondition: OrderByCondition = {};
     this.paginationKeys.forEach((key) => {
-      orderByCondition[`${this.alias}.${key}`] = order;
+      orderByCondition[`${alias}.${key}`] = order;
     });
 
     return orderByCondition;
diff --git a/src/utils.ts b/src/utils.ts
@@ -14,7 +14,7 @@ export function encodeByType(type: string, value: any): string | null {
       return (value as Date).getTime().toString();
     }
     case 'number': {
-      return (value as number).toString();
+      return `${value}`;
     }
     case 'string': {
       return encodeURIComponent(value);
@@ -75,10 +75,6 @@ export function camelOrPascalToUnderscore(str: string): string {
   return str.split(/(?=[A-Z])/).join('_').toLowerCase();
 }
 
-export function camelToUnderscore(str: string): string {
-  return camelOrPascalToUnderscore(str);
-}
-
 export function pascalToUnderscore(str: string): string {
   return camelOrPascalToUnderscore(str);
 }
diff --git a/test/entities/Example.ts b/test/entities/Example.ts
diff --git a/test/entities/Photo.ts b/test/entities/Photo.ts
@@ -0,0 +1,26 @@
+import {
+  Entity,
+  Column,
+  PrimaryGeneratedColumn,
+  ManyToOne,
+} from 'typeorm';
+
+import { User } from './User';
+
+@Entity({ name: 'photos' })
+export class Photo {
+  @PrimaryGeneratedColumn()
+  public id!: number;
+
+  @Column({
+    type: 'text',
+    nullable: false,
+  })
+  public link!: string;
+
+  @ManyToOne(
+    () => User,
+    (user) => user.photos,
+  )
+  public user!: User;
+}
diff --git a/test/entities/User.ts b/test/entities/User.ts
@@ -0,0 +1,36 @@
+import {
+  Entity,
+  Column,
+  PrimaryGeneratedColumn,
+  OneToMany,
+} from 'typeorm';
+
+import { Photo } from './Photo';
+
+@Entity({ name: 'users' })
+export class User {
+  @PrimaryGeneratedColumn()
+  public id!: number;
+
+  @Column({
+    type: 'varchar',
+    length: 255,
+    nullable: false,
+  })
+  public name!: string;
+
+  @Column({
+    type: 'timestamp',
+    nullable: false,
+  })
+  public timestamp!: Date
+
+  @OneToMany(
+    () => Photo,
+    (photo) => photo.user,
+    {
+      cascade: true,
+    },
+  )
+  public photos!: Photo[]
+}
diff --git a/test/integration.ts b/test/integration.ts
@@ -2,8 +2,10 @@ import { expect } from 'chai';
 import { createConnection, getConnection } from 'typeorm';
 
 import { createQueryBuilder } from './utils/createQueryBuilder';
+import { prepareData } from './utils/prepareData';
+import { User } from './entities/User';
+import { Photo } from './entities/Photo';
 import { buildPaginator } from '../src/index';
-import { Example } from './entities/Example';
 
 describe('TypeORM cursor-based pagination test', () => {
   before(async () => {
@@ -14,26 +16,28 @@ describe('TypeORM cursor-based pagination test', () => {
       username: 'test',
       password: 'test',
       database: 'test',
-      entities: [Example],
+      synchronize: true,
+      entities: [User, Photo],
       logging: true,
     });
 
-    await getConnection().query('DROP TABLE IF EXISTS example;');
-    await getConnection().query('CREATE TABLE example as SELECT generate_series(1, 10) AS id;');
+    await prepareData();
   });
 
   it('should paginate correctly with before and after cursor', async () => {
-    const queryBuilder = createQueryBuilder();
+    const queryBuilder = createQueryBuilder().leftJoinAndSelect('user.photos', 'photo');
     const firstPagePaginator = buildPaginator({
-      entity: Example,
+      entity: User,
+      paginationKeys: ['id', 'name', 'timestamp'],
       query: {
         limit: 1,
       },
     });
     const firstPageResult = await firstPagePaginator.paginate(queryBuilder.clone());
 
     const nextPagePaginator = buildPaginator({
-      entity: Example,
+      entity: User,
+      paginationKeys: ['id', 'name', 'timestamp'],
       query: {
         limit: 1,
         afterCursor: firstPageResult.cursor.afterCursor as string,
@@ -42,7 +46,8 @@ describe('TypeORM cursor-based pagination test', () => {
     const nextPageResult = await nextPagePaginator.paginate(queryBuilder.clone());
 
     const prevPagePaginator = buildPaginator({
-      entity: Example,
+      entity: User,
+      paginationKeys: ['id', 'name', 'timestamp'],
       query: {
         limit: 1,
         beforeCursor: nextPageResult.cursor.beforeCursor as string,
@@ -66,14 +71,14 @@ describe('TypeORM cursor-based pagination test', () => {
   it('should return entities with given order', async () => {
     const queryBuilder = createQueryBuilder();
     const ascPaginator = buildPaginator({
-      entity: Example,
+      entity: User,
       query: {
         limit: 1,
         order: 'ASC',
       },
     });
     const descPaginator = buildPaginator({
-      entity: Example,
+      entity: User,
       query: {
         limit: 1,
         order: 'DESC',
@@ -90,7 +95,7 @@ describe('TypeORM cursor-based pagination test', () => {
   it('should return entities with given limit', async () => {
     const queryBuilder = createQueryBuilder();
     const paginator = buildPaginator({
-      entity: Example,
+      entity: User,
       query: {
         limit: 10,
       },
@@ -102,9 +107,9 @@ describe('TypeORM cursor-based pagination test', () => {
   });
 
   it('should return empty array and null cursor if no data', async () => {
-    const queryBuilder = createQueryBuilder().where('example.id > :id', { id: 10 });
+    const queryBuilder = createQueryBuilder().where('user.id > :id', { id: 10 });
     const paginator = buildPaginator({
-      entity: Example,
+      entity: User,
     });
     const result = await paginator.paginate(queryBuilder);
 
@@ -114,7 +119,7 @@ describe('TypeORM cursor-based pagination test', () => {
   });
 
   after(async () => {
-    await getConnection().query('TRUNCATE TABLE example;');
+    await getConnection().query('TRUNCATE TABLE users RESTART IDENTITY CASCADE;');
     await getConnection().close();
   });
 });
diff --git a/test/utils/createQueryBuilder.ts b/test/utils/createQueryBuilder.ts
@@ -1,8 +1,8 @@
 import { getConnection, SelectQueryBuilder } from 'typeorm';
-import { Example } from '../entities/Example';
+import { User } from '../entities/User';
 
-export function createQueryBuilder(): SelectQueryBuilder<Example> {
+export function createQueryBuilder(): SelectQueryBuilder<User> {
   return getConnection()
-    .getRepository(Example)
-    .createQueryBuilder('example');
+    .getRepository(User)
+    .createQueryBuilder('user');
 }
diff --git a/test/utils/prepareData.ts b/test/utils/prepareData.ts
@@ -0,0 +1,18 @@
+import { getConnection } from 'typeorm';
+import { User } from '../entities/User';
+
+export async function prepareData(): Promise<void> {
+  const data = [...Array(10).keys()].map((i) => ({
+    name: `user${i}`,
+    timestamp: new Date(),
+    photos: [
+      {
+        link: `http://photo.com/${i}`,
+      },
+    ],
+  }));
+
+  await getConnection()
+    .getRepository(User)
+    .save(data);
+}

Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ export function encodeByType(type: string, value: any): string \| null {`
`14`	`14`	`return (value as Date).getTime().toString();`
`15`	`15`	`}`
`16`	`16`	`case 'number': {`
`17`		`- return (value as number).toString();`
	`17`	+ return `${value}`;
`18`	`18`	`}`
`19`	`19`	`case 'string': {`
`20`	`20`	`return encodeURIComponent(value);`
`@@ -75,10 +75,6 @@ export function camelOrPascalToUnderscore(str: string): string {`
`75`	`75`	`return str.split(/(?=[A-Z])/).join('_').toLowerCase();`
`76`	`76`	`}`
`77`	`77`
`78`		`-export function camelToUnderscore(str: string): string {`
`79`		`- return camelOrPascalToUnderscore(str);`
`80`		`-}`
`81`		`-`
`82`	`78`	`export function pascalToUnderscore(str: string): string {`
`83`	`79`	`return camelOrPascalToUnderscore(str);`
`84`	`80`	`}`