IF YOU WOULD LIKE TO GET AN ACCOUNT, please write an email to s dot adaszewski at gmail dot com. User accounts are meant only to report issues and/or generate pull requests. This is a purpose-specific Git hosting for ADARED projects. Thank you for your understanding!
Browse Source

Created WBManifestWorkerWrapper and abandoned rootDir transfer back to main thread as it is too slow for big collections, ruins user experience.

master
parent
commit
d5c78ab70f
4 changed files with 55 additions and 60 deletions
  1. +1
    -1
      frontend/rollup.config.js
  2. +29
    -59
      frontend/src/js/component/wb-collection-content.js
  3. +25
    -0
      frontend/src/js/misc/wb-manifest-worker-wrapper.js
  4. +0
    -0
      frontend/src/js/worker/wb-manifest-worker.js

+ 1
- 1
frontend/rollup.config.js View File

@@ -45,7 +45,7 @@ export default {
'node_modules/filesize/lib/filesize.js': 'dist/js/filesize.js',
'node_modules/crypto-js/core.js': 'dist/js/crypto-js/core.js',
'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js',
'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'src/js/worker/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js',
'node_modules/streamsaver/mitm.html': 'dist/mitm.html',
'node_modules/streamsaver/sw.js': 'dist/sw.js',


+ 29
- 59
frontend/src/js/component/wb-collection-content.js View File

@@ -2,9 +2,9 @@ import { h, Component } from 'preact';
import WBTable from 'wb-table';
import WBBreadcrumbs from 'wb-breadcrumbs';
import WBPagination from 'wb-pagination';
import WBRootDirWrapper from 'wb-rootdir-wrapper';
import makeArvadosRequest from 'make-arvados-request';
import wbDownloadFile from 'wb-download-file';
import WBManifestWorkerWrapper from 'wb-manifest-worker-wrapper';
function unescapeName(name) {
return name.replace(/(\\\\|\\[0-9]{3})/g,
@@ -15,14 +15,12 @@ class WBCollectionContent extends Component {
constructor(...args) {
super(...args);
this.state.rows = [];
this.state.manifestWorker = new Worker('/js/wb-manifest-worker.js');
this.state.manifestWorker.onerror = (e => console.log(e));
this.state.manifestWorker = new WBManifestWorkerWrapper();
this.state.loaded = 0;
this.state.total = 0;
this.state.mode = 'manifestDownload';
this.state.parsedStreams = 0;
this.state.totalStreams = 1;
this.state.rootDirWrapper = null;
}
getUrl(params) {
@@ -45,6 +43,7 @@ class WBCollectionContent extends Component {
{ 'onProgress': e => {
this.setState({ 'loaded': e.loaded, 'total': e.total });
} });
prom = prom.then(xhr => {
const streams = xhr.response.manifest_text.split('\n');
const paths = streams.filter(s => s).map(s => {
@@ -55,38 +54,20 @@ class WBCollectionContent extends Component {
let prom_1 = new Promise(accept => accept());
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = () => accept();
manifestWorker.postMessage([ 'precreatePaths', paths ]);
this.setState({
'totalStreams': streams.length,
'parsedStreams': 0,
'mode': 'manifestParse'
})
})
return prom_2;
this.setState({
'totalStreams': streams.length,
'parsedStreams': 0,
'mode': 'manifestParse'
});
return manifestWorker.postMessage([ 'precreatePaths', paths ]);
});
for (let i = 0; i < streams.length; i++) {
prom_1 = prom_1.then(() => manifestWorker.postMessage([ 'parseStream', streams[i] ]));
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = () => accept();
manifestWorker.postMessage([ 'parseStream', streams[i] ]);
});
return prom_2;
});
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = (e) => accept(e);
manifestWorker.postMessage([ 'listDirectory', '.' + this.props.collectionPath, true ]);
if (i % 1000 === 0)
console.log(i + '/' + streams.length);
});
return prom_2;
});
prom_1 = prom_1.then(() => manifestWorker.postMessage([
'listDirectory', '.' + this.props.collectionPath, true
]));
prom_1 = prom_1.then(e => {
this.prepareRows(e.data[1]);
@@ -94,22 +75,9 @@ class WBCollectionContent extends Component {
});
}
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = e => accept(e);
manifestWorker.postMessage([ 'getData' ]);
});
return prom_2;
});
prom_1 = prom_1.then(e => {
this.state.rootDirWrapper = new WBRootDirWrapper(e.data[1], e.data[2]);
this.setState({
prom_1 = prom_1.then(e => this.setState({
'mode': 'browsingReady'
});
this.prepareRows(this.state.rootDirWrapper.listDirectory('.' +
this.props.collectionPath));
});
}));
return prom_1;
});
@@ -118,17 +86,17 @@ class WBCollectionContent extends Component {
componentWillReceiveProps(nextProps) {
this.props = nextProps;
const { rootDirWrapper, mode } = this.state;
const { manifestWorker, mode } = this.state;
const { collectionPath } = this.props;
if (mode === 'browsingReady') {
const listing = rootDirWrapper.listDirectory('.' + collectionPath);
this.prepareRows(listing);
let prom = manifestWorker.postMessage([ 'listDirectory', '.' + collectionPath, true ]);
prom = prom.then(e => this.prepareRows(e.data[1]));
}
}
prepareRows(listing) {
let { rootDirWrapper, mode } = this.state;
let { manifestWorker, mode } = this.state;
let { collectionPath, page, itemsPerPage, app } = this.props;
let { arvHost, arvToken } = app.state;
@@ -151,14 +119,16 @@ class WBCollectionContent extends Component {
( (mode === 'browsingReady') ? (
<div>
<button class="btn btn-outline-primary mx-1" title="Download"
onclick={ () => {
const file = rootDirWrapper.getFile('.' + collectionPath + '/' + item[1]);
const blob = new Blob([
JSON.stringify([ arvHost, arvToken, item[1], file ])
]);
const blocksBlobUrl = URL.createObjectURL(blob);
window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
} }><i class="fas fa-download"></i></button>
onclick={ () => manifestWorker.postMessage([ 'getFile',
'.' + collectionPath + '/' + item[1] ]).then(e => {
const file = e.data[1];
const blob = new Blob([
JSON.stringify([ arvHost, arvToken, item[1], file ])
]);
const blocksBlobUrl = URL.createObjectURL(blob);
window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
}) }><i class="fas fa-download"></i></button>
<button class="btn btn-outline-primary mx-1" title="View"
onclick={ () => {


+ 25
- 0
frontend/src/js/misc/wb-manifest-worker-wrapper.js View File

@@ -0,0 +1,25 @@
class WBManifestWorkerWrapper {
constructor() {
this.worker = new Worker('/js/wb-manifest-worker.js');
this.worker.onmessage = e => this.onMessage(e);
this.worker.onerror = e => console.log(e);
this.queue = [];
}
onMessage(e) {
if (this.queue.length === 0)
throw Error('Unexpected message from worker');
const accept = this.queue.splice(0, 1)[0];
accept(e);
}
postMessage(m) {
const prom = new Promise(accept => {
this.queue.push(accept);
this.worker.postMessage(m);
});
return prom;
}
}
export default WBManifestWorkerWrapper;

frontend/src/js/misc/wb-manifest-worker.js → frontend/src/js/worker/wb-manifest-worker.js View File


Loading…
Cancel
Save