IF YOU WOULD LIKE TO GET AN ACCOUNT, please write an email to s dot adaszewski at gmail dot com. User accounts are meant only to report issues and/or generate pull requests. This is a purpose-specific Git hosting for ADARED projects. Thank you for your understanding!
sadaszewski
/
wba
1
0
Fork 1
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Implemented file download via StreamSaver.

pull/1/head
Stanislaw Adaszewski 4 years ago
parent
17cf0550f4
commit
a6c0a704a9
7 changed files with 236 additions and 44 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -0
      frontend/rollup.config.js
  2. +2
    -0
      frontend/src/html/index.html
  3. +30
    -44
      frontend/src/js/component/wb-collection-content.js
  4. +3
    -0
      frontend/src/js/misc/wb-manifest-worker.js
  5. +82
    -0
      frontend/src/js/misc/wb-rootdir-wrapper.js
  6. +3
    -0
      frontend/src/js/page/wb-app.js
  7. +112
    -0
      frontend/src/js/page/wb-download-page.js

+ 4
- 0
frontend/rollup.config.js View File

@@ -47,6 +47,10 @@ export default {
'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js', 'node_modules/crypto-js/md5.js': 'dist/js/crypto-js/md5.js',
'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js', 'src/js/misc/wb-manifest-worker.js': 'dist/js/wb-manifest-worker.js',
'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js', 'node_modules/js-yaml/dist/js-yaml.min.js': 'dist/js/js-yaml.min.js',
'node_modules/streamsaver/mitm.html': 'dist/mitm.html',
'node_modules/streamsaver/sw.js': 'dist/sw.js',
'node_modules/streamsaver/StreamSaver.js': 'dist/js/StreamSaver.js',
'node_modules/web-streams-polyfill/dist/ponyfill.js': 'dist/js/web-streams-polyfill/ponyfill.js',
verbose: true verbose: true
}), }),
buble({jsx: 'h'}), buble({jsx: 'h'}),


+ 2
- 0
frontend/src/html/index.html View File

@@ -4,6 +4,7 @@
<link rel="stylesheet" type="text/css" href="/css/bootstrap.min.css" /> <link rel="stylesheet" type="text/css" href="/css/bootstrap.min.css" />
<link rel="stylesheet" type="text/css" href="/css/all.min.css" /> <link rel="stylesheet" type="text/css" href="/css/all.min.css" />
<link rel="stylesheet" type="text/css" href="/css/index.css" /> <link rel="stylesheet" type="text/css" href="/css/index.css" />
<script language="javascript" src="/js/web-streams-polyfill/ponyfill.js"></script>
<script language="javascript"> <script language="javascript">
window.process = { 'env': { 'NODE_ENV': 'production' } }; window.process = { 'env': { 'NODE_ENV': 'production' } };
</script> </script>
@@ -15,6 +16,7 @@
<script language="javascript" src="/js/crypto-js/core.js"></script> <script language="javascript" src="/js/crypto-js/core.js"></script>
<script language="javascript" src="/js/crypto-js/md5.js"></script> <script language="javascript" src="/js/crypto-js/md5.js"></script>
<script language="javascript" src="/js/js-yaml.min.js"></script> <script language="javascript" src="/js/js-yaml.min.js"></script>
<script language="javascript" src="/js/StreamSaver.js"></script>
</head> </head>
<body> <body>
<script language="javascript" src="/js/app.min.js"></script> <script language="javascript" src="/js/app.min.js"></script>


+ 30
- 44
frontend/src/js/component/wb-collection-content.js View File

@@ -1,9 +1,8 @@
import { h, Component } from 'preact'; import { h, Component } from 'preact';
import WBTable from 'wb-table'; import WBTable from 'wb-table';
import WBBreadcrumbs from 'wb-breadcrumbs'; import WBBreadcrumbs from 'wb-breadcrumbs';
// import { WBManifestReader } from 'wb-collection-manifest';
// import WBManifestReader from 'wb-manifest-reader';
import WBPagination from 'wb-pagination'; import WBPagination from 'wb-pagination';
import WBRootDirWrapper from 'wb-rootdir-wrapper';
import makeArvadosRequest from 'make-arvados-request'; import makeArvadosRequest from 'make-arvados-request';
import wbDownloadFile from 'wb-download-file'; import wbDownloadFile from 'wb-download-file';
@@ -23,6 +22,7 @@ class WBCollectionContent extends Component {
this.state.mode = 'manifestDownload'; this.state.mode = 'manifestDownload';
this.state.parsedStreams = 0; this.state.parsedStreams = 0;
this.state.totalStreams = 1; this.state.totalStreams = 1;
this.state.rootDirWrapper = null;
} }
getUrl(params) { getUrl(params) {
@@ -95,10 +95,20 @@ class WBCollectionContent extends Component {
} }
prom_1 = prom_1.then(() => { prom_1 = prom_1.then(() => {
const prom_2 = new Promise(accept => {
manifestWorker.onmessage = e => accept(e);
manifestWorker.postMessage([ 'getData' ]);
});
return prom_2;
});
prom_1 = prom_1.then(e => {
this.state.rootDirWrapper = new WBRootDirWrapper(e.data[1], e.data[2]);
this.setState({ this.setState({
'mode': 'browsingReady' 'mode': 'browsingReady'
}); });
this.prepareRows();
this.prepareRows(this.state.rootDirWrapper.listDirectory('.' +
this.props.collectionPath));
}); });
return prom_1; return prom_1;
@@ -106,38 +116,22 @@ class WBCollectionContent extends Component {
} }
componentWillReceiveProps(nextProps) { componentWillReceiveProps(nextProps) {
const { manifestWorker, mode } = this.state;
const { collectionPath } = nextProps;
if (mode === 'browsingReady') {
this.state.mode = 'waitForListing';
let prom = new Promise(accept => {
manifestWorker.onmessage = (e) => accept(e);
manifestWorker.postMessage([ 'listDirectory', '.' + collectionPath ]);
});
this.props = nextProps;
prom = prom.then(e => {
this.state.mode = 'browsingReady';
this.prepareRows(e.data[1]);
});
const { rootDirWrapper, mode } = this.state;
const { collectionPath } = this.props;
if (mode === 'browsingReady') {
const listing = rootDirWrapper.listDirectory('.' + collectionPath);
this.prepareRows(listing);
} }
this.props = nextProps;
// this.prepareRows();
} }
prepareRows(listing) { prepareRows(listing) {
if (listing)
this.state.listing = listing;
else
listing = this.state.listing;
let { manifestReader, mode } = this.state;
let { collectionPath, page, itemsPerPage } = this.props;
let { arvHost, arvToken } = this.props.app.state;
//path = path.split('/');
//path = [ '.' ].concat(path);
let { rootDirWrapper, mode } = this.state;
let { collectionPath, page, itemsPerPage, app } = this.props;
let { arvHost, arvToken } = app.state;
//let listing = manifestReader.listDirectory('.' + collectionPath)
const numPages = Math.ceil(listing.length / itemsPerPage); const numPages = Math.ceil(listing.length / itemsPerPage);
listing = listing.slice(page * itemsPerPage, listing = listing.slice(page * itemsPerPage,
page * itemsPerPage + itemsPerPage); page * itemsPerPage + itemsPerPage);
@@ -158,25 +152,17 @@ class WBCollectionContent extends Component {
<div> <div>
<button class="btn btn-outline-primary mx-1" title="Download" <button class="btn btn-outline-primary mx-1" title="Download"
onclick={ () => { onclick={ () => {
let prom = wbDownloadFile(arvHost, arvToken, manifestReader,
'.' + collectionPath + '/' + item[1]);
prom = prom.then(blocks => {
const blob = new Blob(blocks);
const a = document.createElement('a');
a.name = item[1];
a.href = window.URL.createObjectURL(blob);
a.click();
});
const file = rootDirWrapper.getFile('.' + collectionPath + '/' + item[1]);
const blob = new Blob([
JSON.stringify([ arvHost, arvToken, item[1], file ])
]);
const blocksBlobUrl = URL.createObjectURL(blob);
window.open('/download/' + encodeURIComponent(blocksBlobUrl), '_blank');
} }><i class="fas fa-download"></i></button> } }><i class="fas fa-download"></i></button>
<button class="btn btn-outline-primary mx-1" title="View" <button class="btn btn-outline-primary mx-1" title="View"
onclick={ () => { onclick={ () => {
let prom = wbDownloadFile(arvHost, arvToken, manifestReader,
'.' + collectionPath + '/' + item[1]);
prom = prom.then(blocks => {
const blob = new Blob(blocks);
window.open(window.URL.createObjectURL(blob));
});
alert('Not implemented.')
} }><i class="far fa-eye"></i></button> } }><i class="far fa-eye"></i></button>
</div> </div>
) : null) ) : null)


+ 3
- 0
frontend/src/js/misc/wb-manifest-worker.js View File

@@ -16,6 +16,9 @@ onmessage = function(e) {
const lst = listDirectory(rootDir, e.data[1], e.data[2]); const lst = listDirectory(rootDir, e.data[1], e.data[2]);
postMessage([ 'listDirectoryResult', lst ]) postMessage([ 'listDirectoryResult', lst ])
break; } break; }
case 'getData': {
postMessage([ 'getDataResult', rootDir, streams ]);
break; }
default: default:
throw Error('Unknown verb: ' + e.data[0]); throw Error('Unknown verb: ' + e.data[0]);
} }


+ 82
- 0
frontend/src/js/misc/wb-rootdir-wrapper.js View File

@@ -0,0 +1,82 @@
class WBRootDirWrapper {
constructor(rootDir, streams) {
this.rootDir = rootDir;
this.streams = streams;
}
findDir(path) {
if (typeof(path) === 'string')
path = path.split('/');
if (path[0] !== '.')
throw Error('Path must begin with a dot component');
let dir = this.rootDir;
for (let i = 1; i < path.length; i++) {
if (!(path[i] in dir))
throw Error('Directory not found');
if (dir[path[i]] instanceof Array)
throw Error('Path is a file not directory');
dir = dir[path[i]];
}
return dir;
}
listDirectory(path) {
let dir = this.findDir(path);
let keys = Object.keys(dir);
keys.sort();
let subdirs = keys.filter(k => !(dir[k] instanceof Array));
let files = keys.filter(k => (dir[k] instanceof Array));
let res = subdirs.map(k => [ 'd', k, null ]);
res = res.concat(files.map(k => [ 'f', k, dir[k][1] ]));
return res;
}
unescapeName(name) {
return name.replace(/(\\\\|\\[0-9]{3})/g, (_, $1) => ($1 === '\\\\' ? '\\' : String.fromCharCode(parseInt($1.substr(1), 8))));
}
escapeName(name) {
return name.replace(/ /g, '\\040');
}
getFile(path) {
if (typeof(path) === 'string')
path = path.split('/');
if (path.length < 2)
throw Error('Invalid file path');
const name = path[path.length - 1];
const dir = this.findDir(path.slice(0, path.length - 1));
if (!(name in dir))
throw Error('File not found');
if (!(dir[name] instanceof Array))
throw Error('Path points to a directory not a file');
const streams = this.streams;
let file = dir[name];
file = [ file[0].map(seg => {
const stm = streams[seg[0]];
const used = stm.map(loc => !( loc[2] <= seg[1] || loc[1] >= seg[1] + seg[2] ) );
const start = used.indexOf(true);
const end = used.lastIndexOf(true) + 1;
if (start === -1)
return [];
const res = [];
for (let i = start; i < end; i++) {
const loc = stm[i];
res.push([ loc[0], Math.max(0, seg[1] - loc[1]),
Math.min(loc[2], seg[1] + seg[2] - loc[1]) ]);
}
return res;
}), file[1] ];
file[0] = file[0].reduce((a, b) => a.concat(b));
return file;
}
}
export default WBRootDirWrapper;

+ 3
- 0
frontend/src/js/page/wb-app.js View File

@@ -11,6 +11,7 @@ import WBCollectionBrowse from 'wb-collection-browse';
import WBUsersPage from 'wb-users-page'; import WBUsersPage from 'wb-users-page';
import WBWorkflowView from 'wb-workflow-view'; import WBWorkflowView from 'wb-workflow-view';
import WBLaunchWorkflowPage from 'wb-launch-workflow-page'; import WBLaunchWorkflowPage from 'wb-launch-workflow-page';
import WBDownloadPage from 'wb-download-page';
import arvadosTypeName from 'arvados-type-name'; import arvadosTypeName from 'arvados-type-name';
class WBApp extends Component { class WBApp extends Component {
@@ -91,6 +92,8 @@ class WBApp extends Component {
<WBWorkflowView path="/workflow/:uuid" app={ this } /> <WBWorkflowView path="/workflow/:uuid" app={ this } />
<WBLaunchWorkflowPage path="/workflow-launch/:workflowUuid" app={ this } /> <WBLaunchWorkflowPage path="/workflow-launch/:workflowUuid" app={ this } />
<WBDownloadPage path="/download/:blocksBlobUrl" app={ this } />
</Router> </Router>
); );
} }


+ 112
- 0
frontend/src/js/page/wb-download-page.js View File

@@ -0,0 +1,112 @@
import { h, Component } from 'preact';
import makeArvadosRequest from 'make-arvados-request';
class WBDownloadPage extends Component {
componentDidMount() {
const { app, blocksBlobUrl } = this.props;
const { arvHost, arvToken } = app.state;
let prom = new Promise((accept, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('GET', blocksBlobUrl);
xhr.onreadystatechange = () => {
if (xhr.readyState !== 4)
return;
if (xhr.status !== 200)
reject(xhr);
else
accept(xhr);
};
xhr.responseType = 'blob';
xhr.send();
});
prom = prom.then(xhr => xhr.response.text());
let name, file;
const { streamSaver, location } = window;
streamSaver.mitm = location.protocol + '//' +
location.hostname + (location.port ?
':' + location.port : '') + '/mitm.html';
let fileStream;
let writer;
let done = false;
prom = prom.then(text => {
let _;
[ _, _, name, file ] = JSON.parse(text);
fileStream = streamSaver.createWriteStream(name, {
size: file[1]
});
writer = fileStream.getWriter();
window.onunload = () => {
writer.abort()
};
window.onbeforeunload = evt => {
if (!done) {
evt.returnValue = `Are you sure you want to leave?`;
}
};
const filters = [
['service_type', '=', 'proxy']
];
return makeArvadosRequest(arvHost, arvToken,
'/arvados/v1/keep_services?filters=' +
encodeURIComponent(JSON.stringify(filters)));
});
prom = prom.then(xhr => {
const services = xhr.response.items;
const i = Math.floor(Math.random() * services.length);
const proxy = services[i];
let prom_1 = new Promise(accept => accept());
for (let k = 0; k < file[0].length; k++) {
const loc = file[0][k];
prom_1 = prom_1.then(() => makeArvadosRequest(
proxy.service_host + ':' + proxy.service_port,
arvToken,
'/' + loc[0],
{ 'useSsl': proxy.service_ssl_flag,
'responseType': 'arraybuffer' }
));
prom_1 = prom_1.then(xhr_1 => {
const blk = xhr_1.response.slice(loc[1], loc[2]);
// const r = new Response(blk);
// r.body.pipeTo(fileStream);
writer.write(new Uint8Array(blk));
});
}
return prom_1;
});
prom = prom.then(() => {
writer.close();
done = true;
});
}
render() {
return (
<div class="container-fluid">
<div class="card my-3">
<div class="card-body">
Downloading, please wait...
</div>
</div>
<div class="alert alert-danger" role="alert">
Do not close this window until the download is finished.
</div>
</div>
);
}
}
export default WBDownloadPage;

Write Preview
Loading…
Cancel
Save