diff --git a/frontend/rollup.config.js b/frontend/rollup.config.js
index b69dcb1..1ac0c99 100755
--- a/frontend/rollup.config.js
+++ b/frontend/rollup.config.js
@@ -47,6 +47,10 @@ export default {
'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js',
'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js',
+ 'node_modules/streamsaver/mitm.html': 'dist/mitm.html',
+ 'node_modules/streamsaver/sw.js': 'dist/sw.js',
+ 'node_modules/streamsaver/StreamSaver.js': 'dist/js/StreamSaver.js',
+ 'node_modules/web-streams-polyfill/dist/ponyfill.js': 'dist/js/web-streams-polyfill/ponyfill.js',
verbose: true
}),
buble({jsx: 'h'}),
diff --git a/frontend/src/html/index.html b/frontend/src/html/index.html
index 4c23a3d..5a01586 100755
--- a/frontend/src/html/index.html
+++ b/frontend/src/html/index.html
@@ -4,6 +4,7 @@
+
@@ -15,6 +16,7 @@
+
diff --git a/frontend/src/js/component/wb-collection-content.js b/frontend/src/js/component/wb-collection-content.js
index 6067c2b..593921d 100644
--- a/frontend/src/js/component/wb-collection-content.js
+++ b/frontend/src/js/component/wb-collection-content.js
@@ -1,9 +1,8 @@
import { h, Component } from 'preact';
import WBTable from 'wb-table';
import WBBreadcrumbs from 'wb-breadcrumbs';
-// import { WBManifestReader } from 'wb-collection-manifest';
-// import WBManifestReader from 'wb-manifest-reader';
import WBPagination from 'wb-pagination';
+import WBRootDirWrapper from 'wb-rootdir-wrapper';
import makeArvadosRequest from 'make-arvados-request';
import wbDownloadFile from 'wb-download-file';
@@ -23,6 +22,7 @@ class WBCollectionContent extends Component {
this.state.mode = 'manifestDownload';
this.state.parsedStreams = 0;
this.state.totalStreams = 1;
+ this.state.rootDirWrapper = null;
}
getUrl(params) {
@@ -95,10 +95,20 @@ class WBCollectionContent extends Component {
}
prom_1 = prom_1.then(() => {
+ const prom_2 = new Promise(accept => {
+ manifestWorker.onmessage = e => accept(e);
+ manifestWorker.postMessage([ 'getData' ]);
+ });
+ return prom_2;
+ });
+
+ prom_1 = prom_1.then(e => {
+ this.state.rootDirWrapper = new WBRootDirWrapper(e.data[1], e.data[2]);
this.setState({
'mode': 'browsingReady'
});
- this.prepareRows();
+ this.prepareRows(this.state.rootDirWrapper.listDirectory('.' +
+ this.props.collectionPath));
});
return prom_1;
@@ -106,38 +116,22 @@ class WBCollectionContent extends Component {
}
componentWillReceiveProps(nextProps) {
- const { manifestWorker, mode } = this.state;
- const { collectionPath } = nextProps;
- if (mode === 'browsingReady') {
- this.state.mode = 'waitForListing';
- let prom = new Promise(accept => {
- manifestWorker.onmessage = (e) => accept(e);
- manifestWorker.postMessage([ 'listDirectory', '.' + collectionPath ]);
- });
+ this.props = nextProps;
- prom = prom.then(e => {
- this.state.mode = 'browsingReady';
- this.prepareRows(e.data[1]);
- });
+ const { rootDirWrapper, mode } = this.state;
+ const { collectionPath } = this.props;
+
+ if (mode === 'browsingReady') {
+ const listing = rootDirWrapper.listDirectory('.' + collectionPath);
+ this.prepareRows(listing);
}
- this.props = nextProps;
- // this.prepareRows();
}
prepareRows(listing) {
- if (listing)
- this.state.listing = listing;
- else
- listing = this.state.listing;
-
- let { manifestReader, mode } = this.state;
- let { collectionPath, page, itemsPerPage } = this.props;
- let { arvHost, arvToken } = this.props.app.state;
-
- //path = path.split('/');
- //path = [ '.' ].concat(path);
+ let { rootDirWrapper, mode } = this.state;
+ let { collectionPath, page, itemsPerPage, app } = this.props;
+ let { arvHost, arvToken } = app.state;
- //let listing = manifestReader.listDirectory('.' + collectionPath)
const numPages = Math.ceil(listing.length / itemsPerPage);
listing = listing.slice(page * itemsPerPage,
page * itemsPerPage + itemsPerPage);
@@ -158,25 +152,17 @@ class WBCollectionContent extends Component {
) : null)
diff --git a/frontend/src/js/misc/wb-manifest-worker.js b/frontend/src/js/misc/wb-manifest-worker.js
index e54e8f2..65ba1bd 100644
--- a/frontend/src/js/misc/wb-manifest-worker.js
+++ b/frontend/src/js/misc/wb-manifest-worker.js
@@ -16,6 +16,9 @@ onmessage = function(e) {
const lst = listDirectory(rootDir, e.data[1], e.data[2]);
postMessage([ 'listDirectoryResult', lst ])
break; }
+ case 'getData': {
+ postMessage([ 'getDataResult', rootDir, streams ]);
+ break; }
default:
throw Error('Unknown verb: ' + e.data[0]);
}
diff --git a/frontend/src/js/misc/wb-rootdir-wrapper.js b/frontend/src/js/misc/wb-rootdir-wrapper.js
new file mode 100644
index 0000000..ed8c4b2
--- /dev/null
+++ b/frontend/src/js/misc/wb-rootdir-wrapper.js
@@ -0,0 +1,82 @@
+class WBRootDirWrapper {
+ constructor(rootDir, streams) {
+ this.rootDir = rootDir;
+ this.streams = streams;
+ }
+
+ findDir(path) {
+ if (typeof(path) === 'string')
+ path = path.split('/');
+ if (path[0] !== '.')
+ throw Error('Path must begin with a dot component');
+ let dir = this.rootDir;
+ for (let i = 1; i < path.length; i++) {
+ if (!(path[i] in dir))
+ throw Error('Directory not found');
+ if (dir[path[i]] instanceof Array)
+ throw Error('Path is a file not directory');
+ dir = dir[path[i]];
+ }
+ return dir;
+ }
+
+ listDirectory(path) {
+ let dir = this.findDir(path);
+ let keys = Object.keys(dir);
+ keys.sort();
+ let subdirs = keys.filter(k => !(dir[k] instanceof Array));
+ let files = keys.filter(k => (dir[k] instanceof Array));
+ let res = subdirs.map(k => [ 'd', k, null ]);
+ res = res.concat(files.map(k => [ 'f', k, dir[k][1] ]));
+ return res;
+ }
+
+ unescapeName(name) {
+ return name.replace(/(\\\\|\\[0-9]{3})/g, (_, $1) => ($1 === '\\\\' ? '\\' : String.fromCharCode(parseInt($1.substr(1), 8))));
+ }
+
+ escapeName(name) {
+ return name.replace(/ /g, '\\040');
+ }
+
+ getFile(path) {
+ if (typeof(path) === 'string')
+ path = path.split('/');
+
+ if (path.length < 2)
+ throw Error('Invalid file path');
+
+ const name = path[path.length - 1];
+
+ const dir = this.findDir(path.slice(0, path.length - 1));
+
+ if (!(name in dir))
+ throw Error('File not found');
+
+ if (!(dir[name] instanceof Array))
+ throw Error('Path points to a directory not a file');
+
+ const streams = this.streams;
+ let file = dir[name];
+ file = [ file[0].map(seg => {
+ const stm = streams[seg[0]];
+ const used = stm.map(loc => !( loc[2] <= seg[1] || loc[1] >= seg[1] + seg[2] ) );
+ const start = used.indexOf(true);
+ const end = used.lastIndexOf(true) + 1;
+ if (start === -1)
+ return [];
+ const res = [];
+ for (let i = start; i < end; i++) {
+ const loc = stm[i];
+ res.push([ loc[0], Math.max(0, seg[1] - loc[1]),
+ Math.min(loc[2], seg[1] + seg[2] - loc[1]) ]);
+ }
+ return res;
+ }), file[1] ];
+ file[0] = file[0].reduce((a, b) => a.concat(b));
+
+ return file;
+ }
+}
+
+export default WBRootDirWrapper;
diff --git a/frontend/src/js/page/wb-app.js b/frontend/src/js/page/wb-app.js
index cf119f8..3abdcd8 100644
--- a/frontend/src/js/page/wb-app.js
+++ b/frontend/src/js/page/wb-app.js
@@ -11,6 +11,7 @@ import WBCollectionBrowse from 'wb-collection-browse';
import WBUsersPage from 'wb-users-page';
import WBWorkflowView from 'wb-workflow-view';
import WBLaunchWorkflowPage from 'wb-launch-workflow-page';
+import WBDownloadPage from 'wb-download-page';
import arvadosTypeName from 'arvados-type-name';
class WBApp extends Component {
@@ -91,6 +92,8 @@ class WBApp extends Component {
+
+
);
}
diff --git a/frontend/src/js/page/wb-download-page.js b/frontend/src/js/page/wb-download-page.js
new file mode 100644
index 0000000..8660d64
--- /dev/null
+++ b/frontend/src/js/page/wb-download-page.js
@@ -0,0 +1,112 @@
+import { h, Component } from 'preact';
+import makeArvadosRequest from 'make-arvados-request';
+
+class WBDownloadPage extends Component {
+ componentDidMount() {
+ const { app, blocksBlobUrl } = this.props;
+ const { arvHost, arvToken } = app.state;
+
+ let prom = new Promise((accept, reject) => {
+ const xhr = new XMLHttpRequest();
+ xhr.open('GET', blocksBlobUrl);
+ xhr.onreadystatechange = () => {
+ if (xhr.readyState !== 4)
+ return;
+ if (xhr.status !== 200)
+ reject(xhr);
+ else
+ accept(xhr);
+ };
+ xhr.responseType = 'blob';
+ xhr.send();
+ });
+ prom = prom.then(xhr => xhr.response.text());
+
+ let name, file;
+
+ const { streamSaver, location } = window;
+ streamSaver.mitm = location.protocol + '//' +
+ location.hostname + (location.port ?
+ ':' + location.port : '') + '/mitm.html';
+
+ let fileStream;
+ let writer;
+ let done = false;
+
+ prom = prom.then(text => {
+ let _;
+
+ [ _, _, name, file ] = JSON.parse(text);
+
+ fileStream = streamSaver.createWriteStream(name, {
+ size: file[1]
+ });
+ writer = fileStream.getWriter();
+
+ window.onunload = () => {
+ writer.abort()
+ };
+
+ window.onbeforeunload = evt => {
+ if (!done) {
+ evt.returnValue = `Are you sure you want to leave?`;
+ }
+ };
+
+ const filters = [
+ ['service_type', '=', 'proxy']
+ ];
+
+ return makeArvadosRequest(arvHost, arvToken,
+ '/arvados/v1/keep_services?filters=' +
+ encodeURIComponent(JSON.stringify(filters)));
+ });
+
+ prom = prom.then(xhr => {
+ const services = xhr.response.items;
+ const i = Math.floor(Math.random() * services.length);
+ const proxy = services[i];
+
+ let prom_1 = new Promise(accept => accept());
+ for (let k = 0; k < file[0].length; k++) {
+ const loc = file[0][k];
+ prom_1 = prom_1.then(() => makeArvadosRequest(
+ proxy.service_host + ':' + proxy.service_port,
+ arvToken,
+ '/' + loc[0],
+ { 'useSsl': proxy.service_ssl_flag,
+ 'responseType': 'arraybuffer' }
+ ));
+ prom_1 = prom_1.then(xhr_1 => {
+ const blk = xhr_1.response.slice(loc[1], loc[2]);
+ // const r = new Response(blk);
+ // r.body.pipeTo(fileStream);
+ writer.write(new Uint8Array(blk));
+ });
+ }
+ return prom_1;
+ });
+
+ prom = prom.then(() => {
+ writer.close();
+ done = true;
+ });
+ }
+
+ render() {
+ return (
+
+
+
+ Downloading, please wait...
+
+
+
+ Do not close this window until the download is finished.
+
+