IF YOU WOULD LIKE TO GET AN ACCOUNT, please write an email to s dot adaszewski at gmail dot com. User accounts are meant only to report issues and/or generate pull requests. This is a purpose-specific Git hosting for ADARED projects. Thank you for your understanding!
Ver código fonte

Created WBManifestWorkerWrapper and abandoned rootDir transfer back to main thread as it is too slow for big collections, ruins user experience.

pull/1/head
Stanislaw Adaszewski 4 anos atrás
pai
commit
d5c78ab70f
4 arquivos alterados com 55 adições e 60 exclusões
  1. +1
    -1
      frontend/rollup.config.js
  2. +29
    -59
      frontend/src/js/component/wb-collection-content.js
  3. +25
    -0
      frontend/src/js/misc/wb-manifest-worker-wrapper.js
  4. +0
    -0
      frontend/src/js/worker/wb-manifest-worker.js

+ 1
- 1
frontend/rollup.config.js Ver arquivo

@@ -45,7 +45,7 @@ export default {
'node_modules/filesize/lib/filesize.js': 'dist/js/filesize.js',
'node_modules/crypto-js/core.js': 'dist/js/crypto-js/core.js',
'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js',
'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'src/js/worker/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js',
'node_modules/streamsaver/mitm.html': 'dist/mitm.html',
'node_modules/streamsaver/sw.js': 'dist/sw.js',


+ 29
- 59
frontend/src/js/component/wb-collection-content.js Ver arquivo

@@ -2,9 +2,9 @@ import { h, Component } from 'preact';
import WBTable from 'wb-table';
import WBBreadcrumbs from 'wb-breadcrumbs';
import WBPagination from 'wb-pagination';
import WBRootDirWrapper from 'wb-rootdir-wrapper';
import makeArvadosRequest from 'make-arvados-request';
import wbDownloadFile from 'wb-download-file';
import WBManifestWorkerWrapper from 'wb-manifest-worker-wrapper';
function unescapeName(name) {
return name.replace(/(\\\\|\\[0-9]{3})/g,
@@ -15,14 +15,12 @@ class WBCollectionContent extends Component {
constructor(...args) {
super(...args);
this.state.rows = [];
this.state.manifestWorker = new Worker('/js/wb-manifest-worker.js');
this.state.manifestWorker.onerror = (e => console.log(e));
this.state.manifestWorker = new WBManifestWorkerWrapper();
this.state.loaded = 0;
this.state.total = 0;
this.state.mode = 'manifestDownload';
this.state.parsedStreams = 0;
this.state.totalStreams = 1;
this.state.rootDirWrapper = null;
}
getUrl(params) {
@@ -45,6 +43,7 @@ class WBCollectionContent extends Component {
{ 'onProgress': e => {
this.setState({ 'loaded': e.loaded, 'total': e.total });
} });
prom = prom.then(xhr => {
const streams = xhr.response.manifest_text.split('\n');
const paths = streams.filter(s => s).map(s => {
@@ -55,38 +54,20 @@ class WBCollectionContent extends Component {
let prom_1 = new Promise(accept => accept());
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = () => accept();
manifestWorker.postMessage([ 'precreatePaths', paths ]);
this.setState({
'totalStreams': streams.length,
'parsedStreams': 0,
'mode': 'manifestParse'
})
})
return prom_2;
this.setState({
'totalStreams': streams.length,
'parsedStreams': 0,
'mode': 'manifestParse'
});
return manifestWorker.postMessage([ 'precreatePaths', paths ]);
});
for (let i = 0; i < streams.length; i++) {
prom_1 = prom_1.then(() => manifestWorker.postMessage([ 'parseStream', streams[i] ]));
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = () => accept();
manifestWorker.postMessage([ 'parseStream', streams[i] ]);
});
return prom_2;
});
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = (e) => accept(e);
manifestWorker.postMessage([ 'listDirectory', '.' + this.props.collectionPath, true ]);
if (i % 1000 === 0)
console.log(i + '/' + streams.length);
});
return prom_2;
});
prom_1 = prom_1.then(() => manifestWorker.postMessage([
'listDirectory', '.' + this.props.collectionPath, true
]));
prom_1 = prom_1.then(e => {
this.prepareRows(e.data[1]);
@@ -94,22 +75,9 @@ class WBCollectionContent extends Component {
});
}
prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = e => accept(e);
manifestWorker.postMessage([ 'getData' ]);
});
return prom_2;
});
prom_1 = prom_1.then(e => {
this.state.rootDirWrapper = new WBRootDirWrapper(e.data[1], e.data[2]);
this.setState({
prom_1 = prom_1.then(e => this.setState({
'mode': 'browsingReady'
});
this.prepareRows(this.state.rootDirWrapper.listDirectory('.' +
this.props.collectionPath));
});
}));
return prom_1;
});
@@ -118,17 +86,17 @@ class WBCollectionContent extends Component {
componentWillReceiveProps(nextProps) {
this.props = nextProps;
const { rootDirWrapper, mode } = this.state;
const { manifestWorker, mode } = this.state;
const { collectionPath } = this.props;
if (mode === 'browsingReady') {
const listing = rootDirWrapper.listDirectory('.' + collectionPath);
this.prepareRows(listing);
let prom = manifestWorker.postMessage([ 'listDirectory', '.' + collectionPath, true ]);
prom = prom.then(e => this.prepareRows(e.data[1]));
}
}
prepareRows(listing) {
let { rootDirWrapper, mode } = this.state;
let { manifestWorker, mode } = this.state;
let { collectionPath, page, itemsPerPage, app } = this.props;
let { arvHost, arvToken } = app.state;
@@ -151,14 +119,16 @@ class WBCollectionContent extends Component {
( (mode === 'browsingReady') ? (
<div>
<button class="btn btn-outline-primary mx-1" title="Download"
onclick={ () => {
const file = rootDirWrapper.getFile('.' + collectionPath + '/' + item[1]);
const blob = new Blob([
JSON.stringify([ arvHost, arvToken, item[1], file ])
]);
const blocksBlobUrl = URL.createObjectURL(blob);
window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
} }><i class="fas fa-download"></i></button>
onclick={ () => manifestWorker.postMessage([ 'getFile',
'.' + collectionPath + '/' + item[1] ]).then(e => {
const file = e.data[1];
const blob = new Blob([
JSON.stringify([ arvHost, arvToken, item[1], file ])
]);
const blocksBlobUrl = URL.createObjectURL(blob);
window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
}) }><i class="fas fa-download"></i></button>
<button class="btn btn-outline-primary mx-1" title="View"
onclick={ () => {


+ 25
- 0
frontend/src/js/misc/wb-manifest-worker-wrapper.js Ver arquivo

@@ -0,0 +1,25 @@
class WBManifestWorkerWrapper {
constructor() {
this.worker = new Worker('/js/wb-manifest-worker.js');
this.worker.onmessage = e => this.onMessage(e);
this.worker.onerror = e => console.log(e);
this.queue = [];
}
onMessage(e) {
if (this.queue.length === 0)
throw Error('Unexpected message from worker');
const accept = this.queue.splice(0, 1)[0];
accept(e);
}
postMessage(m) {
const prom = new Promise(accept => {
this.queue.push(accept);
this.worker.postMessage(m);
});
return prom;
}
}
export default WBManifestWorkerWrapper;

frontend/src/js/misc/wb-manifest-worker.js → frontend/src/js/worker/wb-manifest-worker.js Ver arquivo


Carregando…
Cancelar
Salvar