Skip to content

Commit 372f491

Browse files
committed
add downloadStream API
closes andrewrk#53
1 parent 26ef809 commit 372f491

File tree

3 files changed

+95
-0
lines changed

3 files changed

+95
-0
lines changed

README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,23 @@ And these events:
308308
* `'progress'` - emitted when `progressAmount` and `progressTotal`
309309
properties change.
310310

311+
### client.downloadStream(s3Params)
312+
313+
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
314+
315+
* `s3Params`: params to pass to AWS SDK `getObject`.
316+
317+
The difference between using AWS SDK `getObject` and this one:
318+
319+
* This works with a stream only.
320+
321+
If you want retries, progress, or MD5 checking, you must code it yourself.
322+
323+
Returns a `ReadableStream` with these additional events:
324+
325+
* `'httpHeaders' (statusCode, headers)` - contains the HTTP response
326+
headers and status code.
327+
311328
### client.listObjects(params)
312329

313330
See http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjects-property

lib/index.js

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ var MultipartETag = require('./multipart_etag');
1313
var FdSlicer = require('fd-slicer');
1414
var mime = require('mime');
1515
var StreamSink = require('streamsink');
16+
var PassThrough = require('stream').PassThrough;
1617

1718
var MAX_PUTOBJECT_SIZE = 5 * 1024 * 1024 * 1024;
1819
var MAX_DELETE_COUNT = 1000;
@@ -840,6 +841,61 @@ Client.prototype.downloadBuffer = function(s3Params) {
840841
}
841842
};
842843

844+
Client.prototype.downloadStream = function(s3Params) {
845+
var self = this;
846+
var downloadStream = new PassThrough();
847+
s3Params = extend({}, s3Params);
848+
849+
doDownloadWithPend(function(err) {
850+
if (err) downloadStream.emit('error', err);
851+
});
852+
return downloadStream;
853+
854+
function doDownloadWithPend(cb) {
855+
self.s3Pend.go(function(pendCb) {
856+
doTheDownload(function(err) {
857+
pendCb();
858+
cb(err);
859+
});
860+
});
861+
}
862+
863+
function doTheDownload(cb) {
864+
var errorOccurred = false;
865+
var request = self.s3.getObject(s3Params);
866+
var hashCheckPend = new Pend();
867+
request.on('build', function() {
868+
request.httpRequest.headers.Expect = '100-continue';
869+
});
870+
request.on('httpHeaders', function(statusCode, headers, resp) {
871+
if (statusCode >= 300) {
872+
handleError(new Error("http status code " + statusCode));
873+
return;
874+
}
875+
downloadStream.emit('httpHeaders', statusCode, headers, resp);
876+
var httpStream = resp.httpResponse.createUnbufferedStream();
877+
878+
httpStream.on('error', handleError);
879+
880+
downloadStream.on('finish', function() {
881+
if (errorOccurred) return;
882+
cb();
883+
});
884+
885+
httpStream.pipe(downloadStream);
886+
});
887+
888+
request.send(handleError);
889+
890+
function handleError(err) {
891+
if (!err) return;
892+
if (errorOccurred) return;
893+
errorOccurred = true;
894+
cb(err);
895+
}
896+
}
897+
};
898+
843899
function syncDir(self, params, directionIsToS3) {
844900
var ee = new EventEmitter();
845901
var finditOpts = {

test/test.js

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ var fs = require('fs');
88
var mkdirp = require('mkdirp');
99
var crypto = require('crypto');
1010
var rimraf = require('rimraf');
11+
var StreamSink = require('streamsink');
1112
var tempDir = path.join(__dirname, 'tmp');
1213
var localFile = path.join(tempDir, 'random.png');
1314
var remoteRoot = "node-s3-test/";
@@ -188,6 +189,27 @@ describe("s3", function () {
188189
});
189190
});
190191

192+
it("downloadStream", function(done) {
193+
var client = createClient();
194+
var downloadStream = client.downloadStream({Key: remoteFile, Bucket: s3Bucket});
195+
downloadStream.on('error', done);
196+
var gotHttpHeaders = false;
197+
downloadStream.on('httpHeaders', function(statusCode, headers, resp) {
198+
var contentType = headers['content-type'];
199+
assert.strictEqual(contentType, "image/png");
200+
gotHttpHeaders = true;
201+
});
202+
var sink = new StreamSink();
203+
downloadStream.pipe(sink);
204+
sink.on('finish', function() {
205+
var md5sum = crypto.createHash('md5');
206+
md5sum.update(sink.toBuffer());
207+
assert.strictEqual(md5sum.digest('hex'), hexdigest)
208+
assert.ok(gotHttpHeaders);
209+
done();
210+
});
211+
});
212+
191213
it("lists objects", function(done) {
192214
var params = {
193215
recursive: true,

0 commit comments

Comments
 (0)