refactoring

Signed-off-by: Erick Wendel <erick.workspace@gmail.com>

Author: ErickWendel

preclass/annotations.txt

@@ -1,3 +1,4 @@
 https://developer.chrome.com/articles/fetch-streaming-requests/
 https://www.kaggle.com/datasets/danielalbarracinm/list-of-anime-from-1990-to-2022?resource=download
-https://www.tpeczek.com/2019/04/fetch-api-streams-api-ndjson-and-aspnet.html
+https://www.tpeczek.com/2019/04/fetch-api-streams-api-ndjson-and-aspnet.html
+https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_writable_streams

preclass/app/index.js

@@ -1,22 +1,15 @@
 const API_URL = 'http://localhost:3000'
-async function* startConsumingAPI(signal) {
+
+async function consumeAPI(signal) {
   const response = await fetch(API_URL, {
     signal
   })
 
   const reader = response.body
     .pipeThrough(new TextDecoderStream())
     .pipeThrough(parseNDJSON())
-    .getReader()
 
-  let done = false
-  do {
-    const res = await reader.read()
-    done = res.done
-    if (done) break
-    yield res.value
-  }
-  while (!done)
+  return reader
 }
 
 function parseNDJSON() {
@@ -38,37 +31,41 @@ function parseNDJSON() {
   })
 }
 
-function appendToHTML({ title, description,url_anime, image }, el) {
-  const card = `
-  <article>
-    <div class="text">
-      <h3>${title}</h3>
-      <p>${description.slice(0, 100)}</p>
-      <a href="${url_anime}">Here's why</a>
-    </div>
-  </article>
-  `
-  el.innerHTML += card
+function appendToHTML(el) {
+  return new WritableStream({
+    async write({ title, description, url_anime }) {
+      const card = `
+        <article>
+          <div class="text">
+            <h3>[${++counter}] ${title}</h3>
+            <p>${description.slice(0, 100)}</p>
+            <a href="${url_anime}">Here's why</a>
+          </div>
+        </article>
+        `
+      el.innerHTML += card
+    },
+    abort(reason) {
+      console.log('aborted**', reason)
+    }
+  })
 }
 
-const [start, stop, cards] = ['Start', 'Stop', 'cards'].map(id => document.getElementById(id))
+const [
+  start,
+  stop,
+  cards
+] = ['Start', 'Stop', 'cards']
+  .map(
+    id => document.getElementById(id)
+  )
 
 let abortController = new AbortController()
+let counter = 0
+
 start.addEventListener('click', async () => {
-  const it = startConsumingAPI(abortController.signal)
-  let counter = 0
-  try {
-    for await (const item of it) {
-      counter++
-      console.log('item', counter, item.title)
-      appendToHTML(item, cards)
-      // if (counter >= 10) {
-      //   abortController.abort()
-      // }
-    }
-  } catch (error) {
-    if (!error.message.includes('aborted')) throw error
-  }
+  const reader = await consumeAPI(abortController.signal)
+  reader.pipeTo(appendToHTML(cards))
 })
 
 stop.addEventListener('click', () => {

preclass/server/server.js

@@ -1,15 +1,13 @@
-/*
-echo "id,name,desc,age" > big.csv
-for i in `seq 1 1500`; do node -e "process.stdout.write('$i,erick-$i,'+'$i-text'.repeat(1e5)+',$i\n')" >> big.csv;done
-*/
 import { createServer } from 'node:http'
 import { createReadStream } from 'node:fs'
+import { setTimeout } from 'node:timers/promises'
+import { Transform, Readable } from 'node:stream'
+import { TransformStream, WritableStream } from 'node:stream/web'
 import csvtojson from 'csvtojson'
-import { Transform } from 'node:stream'
 
 const PORT = 3000
 // curl -N localhost:3000
-createServer((request, response) => {
+createServer(async (request, response) => {
   const headers = {
     'Access-Control-Allow-Origin': '*',
     'Access-Control-Allow-Methods': '*',
@@ -22,30 +20,40 @@ createServer((request, response) => {
 
   let items = 0
   request.once('close', () => console.log('connection was closed!', items))
-  createReadStream('./animeflv.csv')
-    .pipe(csvtojson())
-    .pipe(Transform({
-      transform(chunk, enc, cb) {
-        items++
-        const d = JSON.parse(chunk)
-        setTimeout(() => {
-          cb(
-            null,
+
+  // const d = (await r.getReader().read()).value
+  // console.log('d', Buffer.from(d).toString('utf8'))
+  Readable.toWeb(createReadStream('./animeflv.csv'))
+    .pipeThrough(Transform.toWeb(csvtojson()))
+    .pipeThrough(
+      new TransformStream({
+        transform(chunk, controller) {
+          const d = JSON.parse(Buffer.from(chunk))
+          controller.enqueue(
             JSON.stringify({
               title: d.title,
               description: d.description,
               url_anime: d.url_anime,
               image: d.image
             }).concat('\n')
           )
-        }, 200)
-
-      }
-    }))
-    .pipe(response)
+        },
+      })
+    )
+    .pipeTo(
+      new WritableStream({
+        async write(chunk) {
+          await setTimeout(200)
+          items++
+          response.write(chunk)
+        },
+        close() {
+          response.end()
+        }
+      })
+    )
 
   response.writeHead(200, headers)
-
 })
   .listen(PORT)
   .on('listening', _ => console.log('server running at ', PORT))