create test runner fail summary

Signed-off-by: shmck <shawn.j.mckay@gmail.com>

src/services/testRunner/index.ts

@@ -2,7 +2,7 @@ import * as T from 'typings'
 import * as TT from 'typings/tutorial'
 import { exec } from '../node'
 import logger from '../logger'
-import parser from './parser'
+import parser, { ParserOutput } from './parser'
 import { debounce, throttle } from './throttle'
 import onError from '../sentry/onError'
 import { clearOutput, addOutput } from './output'
@@ -49,7 +49,7 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 
     const { stdout, stderr } = result
 
-    const tap = parser(stdout || '')
+    const tap: ParserOutput = parser(stdout || '')
 
     addOutput({ channel: logChannelName, text: tap.logs.join('\n'), show: false })
 
@@ -60,6 +60,7 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
         const failSummary = {
           title: firstFail.message || 'Test Failed',
           description: firstFail.details || 'Unknown error',
+          summary: tap.summary,
         }
         callbacks.onFail(position, failSummary)
         const output = formatFailOutput(tap)

src/services/testRunner/parser.test.ts

@@ -6,7 +6,13 @@ describe('parser', () => {
 1..1
 ok 1 - Should pass
 `
-    expect(parser(example)).toEqual({ ok: true, passed: [{ message: 'Should pass' }], failed: [], logs: [] })
+    expect(parser(example)).toEqual({
+      ok: true,
+      passed: [{ message: 'Should pass' }],
+      failed: [],
+      logs: [],
+      summary: { 'Should pass': true },
+    })
   })
   test('should detect multiple successes', () => {
     const example = `
@@ -20,6 +26,10 @@ ok 2 - Should also pass
       passed: [{ message: 'Should pass' }, { message: 'Should also pass' }],
       failed: [],
       logs: [],
+      summary: {
+        'Should pass': true,
+        'Should also pass': true,
+      },
     })
   })
   test('should detect failure if no tests passed', () => {
@@ -170,6 +180,10 @@ at processImmediate (internal/timers.js:439:21)`,
         },
       ],
       logs: ['log 1', 'log 2'],
+      summary: {
+        'package.json should have "express" installed': true,
+        'server should log "Hello World"': false,
+      },
     })
   })
 })

src/services/testRunner/parser.ts

@@ -1,3 +1,5 @@
+import logger from '../logger'
+
 export interface Fail {
   message: string
   details?: string
@@ -14,6 +16,7 @@ export interface ParserOutput {
   passed: Pass[]
   failed: Fail[]
   logs: string[]
+  summary: { [key: string]: boolean }
 }
 
 const r = {
@@ -37,6 +40,7 @@ const parser = (text: string): ParserOutput => {
     passed: [],
     failed: [],
     logs: [],
+    summary: {},
   }
 
   // temporary holder of error detail strings
@@ -58,12 +62,14 @@ const parser = (text: string): ParserOutput => {
     // be optimistic! check for success
     const isPass = detect('pass', line)
     if (!!isPass) {
-      const pass: Pass = { message: isPass[2].trim() }
+      const message = isPass[2].trim()
+      const pass: Pass = { message }
       if (logs.length) {
         pass.logs = logs
         logs = []
       }
       result.passed.push(pass)
+      result.summary[message] = true
       addCurrentDetails()
       continue
     }
@@ -73,12 +79,14 @@ const parser = (text: string): ParserOutput => {
     if (!!isFail) {
       result.ok = false
       addCurrentDetails()
-      const fail: Fail = { message: isFail[2].trim() }
+      const message = isFail[2].trim()
+      const fail: Fail = { message }
       if (logs.length) {
         fail.logs = logs
         logs = []
       }
       result.failed.push(fail)
+      result.summary[message] = false
       continue
     }
 

typings/index.d.ts

@@ -42,6 +42,7 @@ export interface TestStatus {
   type: 'success' | 'warning' | 'error' | 'loading'
   title: string
   content?: string
+  summary?: { [testName: string]: boolean }
   timeout?: number
 }
 
@@ -121,4 +122,5 @@ export interface ProcessEvent {
 export type TestFail = {
   title: string
   description: string
+  summary: { [testName: string]: boolean }
 }

typings/tutorial.d.ts

@@ -45,10 +45,11 @@ export type TutorialSummary = {
 }
 
 export type StepActions = {
-  commands: string[]
+  commands?: string[]
   commits: string[]
-  files: string[]
-  watchers: string[]
+  files?: string[]
+  watchers?: string[]
+  subtasks?: string[]
 }
 
 export interface TutorialTestRunnerConfig {

web-app/src/services/state/actions/testNotify.ts

@@ -22,6 +22,7 @@ const testActions: ActionFunctionMap<CR.MachineContext, CR.MachineEvent> = {
       type: 'warning',
       title: event.payload.fail.title,
       content: event.payload.fail.description,
+      summary: event.payload.fail.summary,
     }),
   }),
   // @ts-ignore