Merge pull request #65 from fantasticit/fix/file

chore: refactor file serve
This commit is contained in:
fantasticit 2022-06-05 03:15:22 +08:00 committed by GitHub
commit 849c6ab91f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 186 additions and 103 deletions

View File

@ -83,37 +83,26 @@ export const uploadFile = async (
const unitPercent = 1 / chunks.length; const unitPercent = 1 / chunks.length;
const progressMap = {}; const progressMap = {};
/** let url = await HttpClient.request<string | undefined>({
* method: FileApiDefinition.initChunk.method,
*/ url: FileApiDefinition.initChunk.client(),
let url = await uploadFileToServer({ params: {
filename, filename,
file: chunks[0], md5,
chunkIndex: 1,
md5,
isChunk: true,
onUploadProgress: (progress) => {
progressMap[1] = progress * unitPercent;
wraponUploadProgress(
Object.keys(progressMap).reduce((a, c) => {
return (a += progressMap[c]);
}, 0)
);
}, },
}); });
if (!url) { if (!url) {
await Promise.all( await Promise.all(
chunks.slice(1).map((chunk, index) => { chunks.map((chunk, index) => {
const currentIndex = 1 + index + 1;
return uploadFileToServer({ return uploadFileToServer({
filename, filename,
file: chunk, file: chunk,
chunkIndex: currentIndex, chunkIndex: index + 1,
md5, md5,
isChunk: true, isChunk: true,
onUploadProgress: (progress) => { onUploadProgress: (progress) => {
progressMap[currentIndex] = progress * unitPercent; progressMap[index] = progress * unitPercent;
wraponUploadProgress( wraponUploadProgress(
Math.min( Math.min(
Object.keys(progressMap).reduce((a, c) => { Object.keys(progressMap).reduce((a, c) => {
@ -136,9 +125,7 @@ export const uploadFile = async (
}, },
}); });
} }
wraponUploadProgress(1); wraponUploadProgress(1);
return url; return url;
} }
}; };

View File

@ -7,6 +7,14 @@ export declare const FileApiDefinition: {
server: "upload"; server: "upload";
client: () => string; client: () => string;
}; };
/**
*
*/
initChunk: {
method: "post";
server: "upload/initChunk";
client: () => string;
};
/** /**
* *
*/ */

View File

@ -10,6 +10,14 @@ exports.FileApiDefinition = {
server: 'upload', server: 'upload',
client: function () { return '/file/upload'; } client: function () { return '/file/upload'; }
}, },
/**
*
*/
initChunk: {
method: 'post',
server: 'upload/initChunk',
client: function () { return '/file/upload/initChunk'; }
},
/** /**
* *
*/ */

View File

@ -8,6 +8,15 @@ export const FileApiDefinition = {
client: () => '/file/upload', client: () => '/file/upload',
}, },
/**
*
*/
initChunk: {
method: 'post' as const,
server: 'upload/initChunk' as const,
client: () => '/file/upload/initChunk',
},
/** /**
* *
*/ */

View File

@ -26,6 +26,16 @@ export class FileController {
return this.fileService.uploadFile(file, query); return this.fileService.uploadFile(file, query);
} }
/**
*
* @param file
*/
@Post(FileApiDefinition.initChunk.server)
@UseGuards(JwtGuard)
initChunk(@Query() query: FileQuery) {
return this.fileService.initChunk(query);
}
/** /**
* *
* @param file * @param file

View File

@ -96,18 +96,31 @@ export class AliyunOssClient extends BaseOssClient {
* @param query * @param query
* @returns * @returns
*/ */
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> { async initChunk(query: FileQuery): Promise<string | void> {
const { md5, filename, chunkIndex } = query; const { md5, filename } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename); const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
if (maybeOssURL) { if (maybeOssURL) {
return maybeOssURL; return maybeOssURL;
} }
return '';
}
/**
*
* FIXME: 阿里云的文档没看懂
* @param file
* @param query
* @returns
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const dir = this.getStoreDir(md5); const dir = this.getStoreDir(md5);
const chunksDir = path.join(dir, 'chunks'); const chunksDir = path.join(dir, 'chunks');
fs.ensureDirSync(chunksDir); fs.ensureDirSync(chunksDir);

View File

@ -69,16 +69,12 @@ export class LocalOssClient extends BaseOssClient {
} }
/** /**
* *
* @param file * @param file
* @param query * @param query
*/ */
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> { async initChunk(query: FileQuery): Promise<void | string> {
const { filename, md5, chunkIndex } = query; const { filename, md5 } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const { absolute, relative } = this.getStoreDir(md5); const { absolute, relative } = this.getStoreDir(md5);
const absoluteFilepath = path.join(absolute, filename); const absoluteFilepath = path.join(absolute, filename);
@ -88,6 +84,22 @@ export class LocalOssClient extends BaseOssClient {
return this.serveFilePath(relativeFilePath); return this.serveFilePath(relativeFilePath);
} }
return '';
}
/**
*
* @param file
* @param query
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const { absolute } = this.getStoreDir(md5);
const chunksDir = path.join(absolute, 'chunks'); const chunksDir = path.join(absolute, 'chunks');
fs.ensureDirSync(chunksDir); fs.ensureDirSync(chunksDir);
fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer); fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer);

View File

@ -8,7 +8,8 @@ export type FileQuery = {
export abstract class OssClient { export abstract class OssClient {
abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>; abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string>; abstract initChunk(query: FileQuery): Promise<void | string>;
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
abstract mergeChunk(query: FileQuery): Promise<string>; abstract mergeChunk(query: FileQuery): Promise<string>;
} }
@ -25,7 +26,12 @@ export class BaseOssClient implements OssClient {
} }
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> { initChunk(query: FileQuery): Promise<void | string> {
throw new Error('Method not implemented.');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');
} }

View File

@ -1,17 +1,34 @@
import * as TencentCos from 'cos-nodejs-sdk-v5'; import * as TencentCos from 'cos-nodejs-sdk-v5';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as path from 'path';
import { BaseOssClient, FileQuery } from './oss.client'; import { BaseOssClient, FileQuery } from './oss.client';
/**
* uploadId
* @param inOssFileName
* @param uploadId
* @returns
*/
function initUploadId(inOssFileName, uploadId) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
fs.ensureFileSync(uploadIdFile);
return fs.writeFileSync(uploadIdFile, uploadId);
}
function getUploadId(inOssFileName) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
return fs.readFileSync(uploadIdFile, 'utf-8');
}
function deleteUploadId(inOssFileName) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
return fs.removeSync(uploadIdFile);
}
export class TencentOssClient extends BaseOssClient { export class TencentOssClient extends BaseOssClient {
private client: TencentCos | null; private client: TencentCos | null;
private uploadIdMap: Map<string, string> = new Map();
private uploadChunkEtagMap: Map<
string,
{
PartNumber: number;
ETag: string;
}[]
> = new Map();
/** /**
* *
@ -109,36 +126,6 @@ export class TencentOssClient extends BaseOssClient {
}); });
} }
/**
*
* @param inOssFileName
* @returns
*/
private getUploadChunkId(inOssFileName): Promise<string> {
if (this.uploadIdMap.has(inOssFileName)) {
return Promise.resolve(this.uploadIdMap.get(inOssFileName));
}
return new Promise((resolve, reject) => {
const params = {
Bucket: this.configService.get('oss.tencent.config.Bucket'),
Region: this.configService.get('oss.tencent.config.Region'),
Key: inOssFileName,
};
this.ensureOssClient();
this.client.multipartInit(params, (err, data) => {
if (err) {
reject(err);
} else {
const uploadId = data.UploadId;
this.uploadIdMap.set(inOssFileName, uploadId);
this.uploadChunkEtagMap.set(uploadId, []);
resolve(uploadId);
}
});
});
}
/** /**
* *
* @param uploadId * @param uploadId
@ -158,14 +145,10 @@ export class TencentOssClient extends BaseOssClient {
Body: file.buffer, Body: file.buffer,
}; };
this.ensureOssClient(); this.ensureOssClient();
this.client.multipartUpload(params, (err, data) => { this.client.multipartUpload(params, (err) => {
if (err) { if (err) {
reject(err); reject(err);
} else { } else {
this.uploadChunkEtagMap.get(uploadId).push({
PartNumber: chunkIndex,
ETag: data.ETag,
});
resolve(); resolve();
} }
}); });
@ -188,26 +171,38 @@ export class TencentOssClient extends BaseOssClient {
Key: inOssFileName, Key: inOssFileName,
}; };
this.ensureOssClient(); this.ensureOssClient();
const parts = this.uploadChunkEtagMap.get(uploadId);
parts.sort((a, b) => a.PartNumber - b.PartNumber);
this.client.multipartComplete( this.client.multipartListPart(
{ {
...params, ...params,
UploadId: uploadId, UploadId: uploadId,
Parts: parts,
}, },
(err) => { (err, data) => {
if (err) { if (err) {
reject(err); reject(err);
} else { } else {
this.client.getObjectUrl(params, (err, data) => { const parts = data.Part;
if (err) {
reject(err); this.client.multipartComplete(
} else { {
resolve(data.Url); ...params,
UploadId: uploadId,
Parts: parts,
},
(err) => {
if (err) {
reject(err);
} else {
this.client.getObjectUrl(params, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data.Url);
}
});
}
} }
}); );
} }
} }
); );
@ -234,13 +229,52 @@ export class TencentOssClient extends BaseOssClient {
return res as string; return res as string;
} }
/**
*
* @param file
* @param query
* @returns
*/
async initChunk(query: FileQuery): Promise<string | void> {
const { md5, filename } = query;
this.ensureOssClient();
const inOssFileName = this.getInOssFileName(md5, filename);
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
if (maybeOssURL) {
return maybeOssURL as string;
}
const params = {
Bucket: this.configService.get('oss.tencent.config.Bucket'),
Region: this.configService.get('oss.tencent.config.Region'),
Key: inOssFileName,
};
const promise = new Promise((resolve, reject) => {
this.client.multipartInit(params, (err, data) => {
if (err) {
reject(err);
} else {
const uploadId = data.UploadId;
initUploadId(inOssFileName, uploadId);
resolve(uploadId);
}
});
});
await promise;
return '';
}
/** /**
* *
* @param file * @param file
* @param query * @param query
* @returns * @returns
*/ */
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> { async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, filename, chunkIndex } = query; const { md5, filename, chunkIndex } = query;
if (!('chunkIndex' in query)) { if (!('chunkIndex' in query)) {
@ -249,15 +283,8 @@ export class TencentOssClient extends BaseOssClient {
this.ensureOssClient(); this.ensureOssClient();
const inOssFileName = this.getInOssFileName(md5, filename); const inOssFileName = this.getInOssFileName(md5, filename);
const uploadId = getUploadId(inOssFileName);
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
if (maybeOssURL) {
return maybeOssURL as string;
}
const uploadId = await this.getUploadChunkId(inOssFileName);
await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file); await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file);
return '';
} }
/** /**
@ -268,10 +295,9 @@ export class TencentOssClient extends BaseOssClient {
async mergeChunk(query: FileQuery): Promise<string> { async mergeChunk(query: FileQuery): Promise<string> {
const { filename, md5 } = query; const { filename, md5 } = query;
const inOssFileName = this.getInOssFileName(md5, filename); const inOssFileName = this.getInOssFileName(md5, filename);
const uploadId = await this.getUploadChunkId(inOssFileName); const uploadId = getUploadId(inOssFileName);
const data = await this.completeUploadChunkToCos(uploadId, inOssFileName); const data = await this.completeUploadChunkToCos(uploadId, inOssFileName);
this.uploadIdMap.delete(inOssFileName); deleteUploadId(inOssFileName);
this.uploadChunkEtagMap.delete(uploadId);
return data; return data;
} }
} }

View File

@ -14,6 +14,10 @@ export class FileService {
return this.ossClient.uploadFile(file, query); return this.ossClient.uploadFile(file, query);
} }
async initChunk(query) {
return this.ossClient.initChunk(query);
}
async uploadChunk(file, query) { async uploadChunk(file, query) {
return this.ossClient.uploadChunk(file, query); return this.ossClient.uploadChunk(file, query);
} }