What is the best way to download a big file in NodeJS?
var request = require('request')
request('http://localhost:11146/base/bigFile.zip', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
UPDATE:
request
module is deprecated
It is recommended to use active modules. My personal preference is got
Please refer got streams for examples.
Another alternative is node-fetch
Easiest way is to use request
module
Here you are trying to store entire result in memory and console log it. 5GB is pretty much large, either you must increase Node.js memory limit (not recommended) or you must use streams. See the streaming example below from request
npm documentation:
const fs = require('fs');
const request = require('request');
request('http://google.com/doodle.png').pipe(fs.createWriteStream('doodle.png'))
You must pipe the response, so that whether is 1MB or 1GB or 1TB, only a fraction of the file will be in memory and it will be written to disk as soon as possible. You can use the same approach using Node.js built in functions, but implementation will be difficult and be like re-inventing the wheel when request
module is there.
For download with progress you can use request-progress
module along with request
module, see the example below (taken from their documentation):
var fs = require('fs');
var request = require('request');
var progress = require('request-progress');
// The options argument is optional so you can omit it
progress(request('https://az412801.vo.msecnd.net/vhd/VMBuild_20141027/VirtualBox/IE11/Windows/IE11.Win8.1.For.Windows.VirtualBox.zip'), {
// throttle: 2000, // Throttle the progress event to 2000ms, defaults to 1000ms
// delay: 1000, // Only start to emit after 1000ms delay, defaults to 0ms
// lengthHeader: 'x-transfer-length' // Length header to use, defaults to content-length
})
.on('progress', function (state) {
// The state is an object that looks like this:
// {
// percent: 0.5, // Overall percent (between 0 to 1)
// speed: 554732, // The download speed in bytes/sec
// size: {
// total: 90044871, // The total payload size in bytes
// transferred: 27610959 // The transferred payload size in bytes
// },
// time: {
// elapsed: 36.235, // The total elapsed seconds since the start (3 decimals)
// remaining: 81.403 // The remaining seconds to finish (3 decimals)
// }
// }
console.log('progress', state);
})
.on('error', function (err) {
// Do something with err
})
.on('end', function () {
// Do something after request finishes
})
.pipe(fs.createWriteStream('IE11.Win8.1.For.Windows.VirtualBox.zip'));