You may face a situation where your application need to handle file upload and store the file on a cloud storage. Usually the cloud storage service provides APIs that allow customer to upload files. If you're using Alibaba Cloud OSS (Object Storage Service) and need to upload files (objects) to the service, you come to the right place.
I assume you've already registered for an Alibaba Cloud account (including confirm email address and set up billing information) and created an AccessKey. In case you haven't done those steps, please read how to getting started with Cloud OSS + Node.js.
I'll show three ways to upload a file: standard upload, upload as a stream and multipart upload with retry
First, we create a new helper file for Alibaba OSS.
helpers/alibaba-oss.js
const _ = require('lodash');
const OSS = require('ali-oss');
const fs = require('fs');
function AliOssClient({ region, bucket }) {
if (!(this instanceof AliOssClient)) {
return new AliOssClient();
}
this.client = new OSS({
// Add access key id and secret to .env and load it with the like of dotenv beforehand
accessKeyId: process.env.ALIBABA_CLOUD_ACCESS_KEY_ID,
accessKeySecret: process.env.ALIBABA_CLOUD_ACCESS_KEY_SECRET,
bucket,
region,
});
}Then, we create an instance of it in another file
example.js
const ossclient = new AliOssClient({ bucket: 'your-bucket', region: 'your-bucket-region (e.g. "oss-us-west-1")' });1. Standard Upload
This is the most standard and basic way to upload a file. Add the following code to the helper file
helpers/alibaba-oss.js
/**
* Upload an object to OSS.
* @param {String} objectKey - Object name in OSS
* @param {String} file - Path to local file, buffer, or ReadStream content instance
* @param {Object} [options] - Optional params
* @param {Number} [options.timeout] - Request timeout
* @param {String} [options.mime] - Custom mime
* @param {Object} [options.meta] - user meta, will send with x-oss-meta- prefix string e.g.: { uid: 111, pid: 222 }
* @param {Object} [options.callback] - Callback parameter
* @param {String} options.callback.url - After file uploaded, OSS will send callback to this URL.
* @param {String} [options.callback.host] - Host header for callback request.
* @param {String} options.callback.body - Request body for callback request. e.g.: key=$(key)&etag=$(etag)&my_var=$(x:my_var)
* @param {String} [options.callback.contentType] - Content-Type of the callback requests initiatiated,
* either application/x-www-form-urlencoded (default) or application/json
* @param {String} [options.callback.customValue] - Map of key-values. e.g.: {var1: 'value1', var2: 'value2'}
* @param {Object} [options.headers] - See RFC 2616
* @return {Promise}
*/
AliOssClient.prototype.uploadObject = async function (objectKey, file, options) {
return await this.client.put(objectKey, file, options);
};Then, we call the function above.
example.js
ossClient.uploadObject('object-name.png', '/path/to/local/file/example.png');In case you need to handle a request containing the file to be uploaded, here is the example which uses express framework and multer as the middleware.
const express = require('express');
const Multer = require('multer');
const router = express.Router();
const multer = Multer({
storage: Multer.MemoryStorage,
limits: {
fileSize: 10 * 1024 * 1024, // Maximum file size is 10MB
},
});
router.post(
'/upload-alibaba',
multer.single('image'),
async (req, res, next) => {
try {
const ossClient = new AlibabOss();
const result = await ossClient.uploadBuffer('woolha-alibaba', 'something.png', req.file.buffer);
console.log(result);
return res.send('ok');
} catch (err) {
console.error(err);
return res.status(500).send('Unable to upload');
}
},
);2. Upload Stream
You may want to upload the file in a stream. For this case, we can use client.putStream. By default, it uses chunked encoding. But if you specify contentLength, chunked encoding won't be used. Add the code below to the helper file
helpers/alibaba-oss.js
/**
* Upload stream object to bucket
* @param {String} objectKey
* @param {String} file - Path to file to be uploaded
* @param {Object} [options] - Additonal params
* @param {Number} [options.timeout] - Request timeout
* @param {Object} [options.meta] - User meta, will send with x-oss-meta- prefix string
* @param {String} [options.mime] - Custom mime , will send with Content-Type entity header
* @param {Object} [options.callback] - Callback parameter
* @param {String} options.callback.url - After file uploaded, OSS will send callback to this URL.
* @param {String} [options.callback.host] - Host header for callback request.
* @param {String} options.callback.body - Request body for callback request. e.g.: key=$(key)&etag=$(etag)&my_var=$(x:my_var)
* @param {String} [options.callback.contentType] - Content-Type of the callback requests initiatiated,
* either application/x-www-form-urlencoded (default) or application/json
* @param {String} [options.callback.customValue] - Map of key-values. e.g.: {var1: 'value1', var2: 'value2'}
* @param {Object} [options.headers] - See RFC 2616
* @return {Promise}
*/
AliOssClient.prototype.uploadStreamObject = async function (objectKey, file, options = {}) {
_.defaults(options, {
useChunkedEncoding: true,
});
const stream = fs.createReadStream(file);
if (options.useChunkedEncoding) {
return await this.client.putStream(objectKey, stream, options);
} else {
const size = fs.statSync(file).size;
return await this.client.putStream(objectKey, stream, _.assign(options, { contentLength: size }));
}
};example.js
This is the exmple usage that uses chunked encoding.
ossClient.uploadStreamObject('object-name.png', '/path/to/local/file/example.png', { useChunkedEncoding: false });And this one doesn't use chunked encoding
example.js
ossClient.uploadStreamObject('object-name.png', '/path/to/local/file/example.png');3. Upload Multipart with Retry
If the file to be uploaded is very large, it may take a few minutes for completing the upload. While waiting, it would be nice if we can get the progress (percentage) of the upload process. Another problem with uploading a large file is the failed possibility is much bigger, especially if the network connection is unstable. As a solution, we can perform a multipart upload using client.multipartUpload. In addition, if upload failed, we can make it try again to upload. Of course we don't want to restart the upload from the beginning, so we need a checkpoint updated after each parts successfully uploaded.
Here is the function that handles both multipart and resumable upload. Add it on the helper file
helpers/alibaba-oss.js
/**
*
* @param {String} objectKey - Object name in OSS
* @param {String|File|Blob} - File to be uploaded
* @param {Object} [options] - Optional params
* @param {Number} [options.parallel] - Number of parts uploaded in parallel
* @param {Number} [options.partSize] - Size for each part.
* @param {Function} [options.progress] - Progress callback with 3 parameters:
* ({Number} percentage, {Object} checkpoint, {Object} res)
* @param {Object} [options.checkpoint] - Used to resume upload
* @param {File} options.checkpoint.file - File to be uploaded
* @param {String} options.checkpoint.name - Object name in OSS
* @param {Number} options.checkpoint.fileSize - File size
* @param {Number} options.checkpoint.partSize - Part size
* @param {String} options.checkpoint.uploadId - Upload ID
* @param {Array} options.checkpoint.doneParts - Parts that have been uploaded
* @param {File} options.checkpoint.file - File to be uploaded
* @param {Object} [options.meta] - User meta, will send with x-oss-meta- prefix string
* @param {String} [options.mime] - Custom mime , will send with Content-Type entity header
* @param {Object} [options.callback] - Callback parameter
* @param {String} options.callback.url - After file uploaded, OSS will send callback to this URL.
* @param {String} [options.callback.host] - Host header for callback request.
* @param {String} options.callback.body - Request body for callback request. e.g.: key=$(key)&etag=$(etag)&my_var=$(x:my_var)
* @param {String} [options.callback.contentType] - Content-Type of the callback requests initiatiated,
* either application/x-www-form-urlencoded (default) or application/json
* @param {String} [options.callback.customValue] - Map of key-values. e.g.: {var1: 'value1', var2: 'value2'}
* @param {Object} [options.headers] - See RFC 2616
* @param {Number} [options.timeout] - Request timeout
* @param {Number} [options.maxRetry] - Maximum number of retry
*
* @return {Promise}
*/
AliOssClient.prototype.resumableMultipartUpload = async function (objectKey, file, options = {}) {
const DEFAULT_MAX_RETRY = 3;
const maxRetry = options.maxRetry || DEFAULT_MAX_RETRY;
let retryCount = 0;
let checkpoint;
const upload = async () => {
if (retryCount > maxRetry) {
return Promise.reject(new Error('Unable to upload file'));
}
try {
const result = await this.client.multipartUpload(objectKey, localPath, {
checkpoint,
progress: function (percentage, _checkpoint) {
console.log(`Progress: ${percentage * 100}%`);
checkpoint = _checkpoint;
},
meta: options.meta,
});
console.log(result);
const head = await this.client.head(objectKey);
console.log(head);
return result;
} catch (err) {
console.error(err);
retryCount += 1;
return upload();
}
};
return upload();
};This is the example usage of the function above.
ossClient.resumableMultipartUpload(
'object-multipart.png',
'/home/ivan/Videos/video.mp4',
{
year: 2018,
anything: 'abcde',
maxRetry: 5,
},
);That's all about how to upload file to Alibaba Cloud OSS using Node.js.
