Created WBManifestWorkerWrapper and abandoned rootDir transfer back to main thread as it is too slow for big collections, ruins user experience.

frontend/rollup.config.js

@@ -45,7 +45,7 @@ export default {
       'node_modules/filesize/lib/filesize.js': 'dist/js/filesize.js',
       'node_modules/crypto-js/core.js': 'dist/js/crypto-js/core.js',
       'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js',
-      'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
+      'src/js/worker/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
       'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js',
       'node_modules/streamsaver/mitm.html': 'dist/mitm.html',
       'node_modules/streamsaver/sw.js': 'dist/sw.js',

frontend/src/js/component/wb-collection-content.js

@@ -2,9 +2,9 @@ import { h, Component } from 'preact';
 import WBTable from 'wb-table';
 import WBBreadcrumbs from 'wb-breadcrumbs';
 import WBPagination from 'wb-pagination';
-import WBRootDirWrapper from 'wb-rootdir-wrapper';
 import makeArvadosRequest from 'make-arvados-request';
 import wbDownloadFile from 'wb-download-file';
+import WBManifestWorkerWrapper from 'wb-manifest-worker-wrapper';
 
 function unescapeName(name) {
   return name.replace(/(\\\\|\\[0-9]{3})/g,
@@ -15,14 +15,12 @@ class WBCollectionContent extends Component {
   constructor(...args) {
     super(...args);
     this.state.rows = [];
-    this.state.manifestWorker = new Worker('/js/wb-manifest-worker.js');
-    this.state.manifestWorker.onerror = (e => console.log(e));
+    this.state.manifestWorker = new WBManifestWorkerWrapper();
     this.state.loaded = 0;
     this.state.total = 0;
     this.state.mode = 'manifestDownload';
     this.state.parsedStreams = 0;
     this.state.totalStreams = 1;
-    this.state.rootDirWrapper = null;
   }
 
   getUrl(params) {
@@ -45,6 +43,7 @@ class WBCollectionContent extends Component {
       { 'onProgress': e => {
         this.setState({ 'loaded': e.loaded, 'total': e.total });
       } });
+
     prom = prom.then(xhr => {
       const streams = xhr.response.manifest_text.split('\n');
       const paths = streams.filter(s => s).map(s => {
@@ -55,38 +54,20 @@ class WBCollectionContent extends Component {
       let prom_1 = new Promise(accept => accept());
 
       prom_1 = prom_1.then(() => {
-        const prom_2 = new Promise(accept => {
-          manifestWorker.onmessage = () => accept();
-          manifestWorker.postMessage([ 'precreatePaths', paths ]);
-          this.setState({
-            'totalStreams': streams.length,
-            'parsedStreams': 0,
-            'mode': 'manifestParse'
-          })
-        })
-        return prom_2;
+        this.setState({
+          'totalStreams': streams.length,
+          'parsedStreams': 0,
+          'mode': 'manifestParse'
+        });
+        return manifestWorker.postMessage([ 'precreatePaths', paths ]);
       });
 
       for (let i = 0; i < streams.length; i++) {
+        prom_1 = prom_1.then(() => manifestWorker.postMessage([ 'parseStream', streams[i] ]));
 
-        prom_1 = prom_1.then(() => {
-          const prom_2 = new Promise(accept => {
-            manifestWorker.onmessage = () => accept();
-            manifestWorker.postMessage([ 'parseStream', streams[i] ]);
-          });
-
-          return prom_2;
-        });
-
-        prom_1 = prom_1.then(() => {
-          const prom_2 = new Promise(accept => {
-            manifestWorker.onmessage = (e) => accept(e);
-            manifestWorker.postMessage([ 'listDirectory', '.' + this.props.collectionPath, true ]);
-            if (i % 1000 === 0)
-              console.log(i + '/' + streams.length);
-          });
-          return prom_2;
-        });
+        prom_1 = prom_1.then(() => manifestWorker.postMessage([
+          'listDirectory', '.' + this.props.collectionPath, true
+        ]));
 
         prom_1 = prom_1.then(e => {
           this.prepareRows(e.data[1]);
@@ -94,22 +75,9 @@ class WBCollectionContent extends Component {
         });
       }
 
-      prom_1 = prom_1.then(() => {
-        const prom_2 = new Promise(accept => {
-          manifestWorker.onmessage = e => accept(e);
-          manifestWorker.postMessage([ 'getData' ]);
-        });
-        return prom_2;
-      });
-
-      prom_1 = prom_1.then(e => {
-        this.state.rootDirWrapper = new WBRootDirWrapper(e.data[1], e.data[2]);
-        this.setState({
+      prom_1 = prom_1.then(e =>  this.setState({
           'mode': 'browsingReady'
-        });
-        this.prepareRows(this.state.rootDirWrapper.listDirectory('.' +
-          this.props.collectionPath));
-      });
+      }));
 
       return prom_1;
     });
@@ -118,17 +86,17 @@ class WBCollectionContent extends Component {
   componentWillReceiveProps(nextProps) {
     this.props = nextProps;
 
-    const { rootDirWrapper, mode } = this.state;
+    const { manifestWorker, mode } = this.state;
     const { collectionPath } = this.props;
 
     if (mode === 'browsingReady') {
-      const listing = rootDirWrapper.listDirectory('.' + collectionPath);
-      this.prepareRows(listing);
+      let prom = manifestWorker.postMessage([ 'listDirectory', '.' + collectionPath, true ]);
+      prom = prom.then(e => this.prepareRows(e.data[1]));
     }
   }
 
   prepareRows(listing) {
-    let { rootDirWrapper, mode } = this.state;
+    let { manifestWorker, mode } = this.state;
     let { collectionPath, page, itemsPerPage, app } = this.props;
     let { arvHost, arvToken } = app.state;
 
@@ -151,14 +119,16 @@ class WBCollectionContent extends Component {
           ( (mode === 'browsingReady') ? (
             <div>
               <button class="btn btn-outline-primary mx-1" title="Download"
-                onclick={ () => {
-                  const file = rootDirWrapper.getFile('.' + collectionPath + '/' + item[1]);
-                  const blob = new Blob([
-                    JSON.stringify([ arvHost, arvToken, item[1], file ])
-                  ]);
-                  const blocksBlobUrl = URL.createObjectURL(blob);
-                  window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
-                } }><i class="fas fa-download"></i></button>
+                onclick={ () => manifestWorker.postMessage([ 'getFile',
+                    '.' + collectionPath + '/' + item[1] ]).then(e => {
+                      const file = e.data[1];
+                      const blob = new Blob([
+                        JSON.stringify([ arvHost, arvToken, item[1], file ])
+                      ]);
+                      const blocksBlobUrl = URL.createObjectURL(blob);
+                      window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
+
+                    }) }><i class="fas fa-download"></i></button>
 
               <button class="btn btn-outline-primary mx-1" title="View"
                 onclick={ () => {

frontend/src/js/misc/wb-manifest-worker-wrapper.js

@@ -0,0 +1,25 @@
+class WBManifestWorkerWrapper {
+  constructor() {
+    this.worker = new Worker('/js/wb-manifest-worker.js');
+    this.worker.onmessage = e => this.onMessage(e);
+    this.worker.onerror = e => console.log(e);
+    this.queue = [];
+  }
+
+  onMessage(e) {
+    if (this.queue.length === 0)
+      throw Error('Unexpected message from worker');
+    const accept = this.queue.splice(0, 1)[0];
+    accept(e);
+  }
+
+  postMessage(m) {
+    const prom = new Promise(accept => {
+      this.queue.push(accept);
+      this.worker.postMessage(m);
+    });
+    return prom;
+  }
+}
+
+export default WBManifestWorkerWrapper;