gio
/
think
mirror of https://github.com/fantasticit/think.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #65 from fantasticit/fix/file 

chore: refactor file serve
This commit is contained in:
fantasticit 2022-06-05 03:15:22 +08:00 committed by GitHub
parent 610367b8ae 9983bd65b9
commit 849c6ab91f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 186 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
packages/client/src/services/file.ts
View File

@ -83,37 +83,26 @@ export const uploadFile = async (
const unitPercent = 1 / chunks.length;
const progressMap = {};
/**
* 先上传一块分块，如果文件已上传，即无需上传后续分块
*/
let url = await uploadFileToServer({
filename,
file: chunks[0],
chunkIndex: 1,
md5,
isChunk: true,
onUploadProgress: (progress) => {
progressMap[1] = progress * unitPercent;
wraponUploadProgress(
Object.keys(progressMap).reduce((a, c) => {
return (a += progressMap[c]);
}, 0)
);
let url = await HttpClient.request<string | undefined>({
method: FileApiDefinition.initChunk.method,
url: FileApiDefinition.initChunk.client(),
params: {
filename,
md5,
},
});
if (!url) {
await Promise.all(
chunks.slice(1).map((chunk, index) => {
const currentIndex = 1 + index + 1;
chunks.map((chunk, index) => {
return uploadFileToServer({
filename,
file: chunk,
chunkIndex: currentIndex,
chunkIndex: index + 1,
md5,
isChunk: true,
onUploadProgress: (progress) => {
progressMap[currentIndex] = progress * unitPercent;
progressMap[index] = progress * unitPercent;
wraponUploadProgress(
Math.min(
Object.keys(progressMap).reduce((a, c) => {
@ -136,9 +125,7 @@ export const uploadFile = async (
},
});
}
wraponUploadProgress(1);
return url;
}
};

8
packages/domains/lib/api/file.d.ts vendored
View File

@ -7,6 +7,14 @@ export declare const FileApiDefinition: {
server: "upload";
client: () => string;
};
/**
* 上传分块文件
*/
initChunk: {
method: "post";
server: "upload/initChunk";
client: () => string;
};
/**
* 上传分块文件
*/

8
packages/domains/lib/api/file.js
View File

@ -10,6 +10,14 @@ exports.FileApiDefinition = {
server: 'upload',
client: function () { return '/file/upload'; }
},
/**
* 上传分块文件
*/
initChunk: {
method: 'post',
server: 'upload/initChunk',
client: function () { return '/file/upload/initChunk'; }
},
/**
* 上传分块文件
*/

9
packages/domains/src/api/file.ts
View File

@ -8,6 +8,15 @@ export const FileApiDefinition = {
client: () => '/file/upload',
},
/**
* 初始分块上传
*/
initChunk: {
method: 'post' as const,
server: 'upload/initChunk' as const,
client: () => '/file/upload/initChunk',
},
/**
* 上传分块文件
*/

10
packages/server/src/controllers/file.controller.ts
View File

@ -26,6 +26,16 @@ export class FileController {
return this.fileService.uploadFile(file, query);
}
/**
* 初始分块文件
* @param file
*/
@Post(FileApiDefinition.initChunk.server)
@UseGuards(JwtGuard)
initChunk(@Query() query: FileQuery) {
return this.fileService.initChunk(query);
}
/**
* 上传分块文件
* @param file

25
packages/server/src/helpers/file.helper/aliyun.client.ts
View File

@ -96,18 +96,31 @@ export class AliyunOssClient extends BaseOssClient {
* @param query
* @returns
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
const { md5, filename, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
async initChunk(query: FileQuery): Promise<string | void> {
const { md5, filename } = query;
const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
if (maybeOssURL) {
return maybeOssURL;
}
return '';
}
/**
* 将切片临时存储到服务器
* FIXME: 阿里云的文档没看懂，故做成这种服务器中转的蠢模式
* @param file
* @param query
* @returns
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const dir = this.getStoreDir(md5);
const chunksDir = path.join(dir, 'chunks');
fs.ensureDirSync(chunksDir);

26
packages/server/src/helpers/file.helper/local.client.ts
View File

@ -69,16 +69,12 @@ export class LocalOssClient extends BaseOssClient {
}
/**
* 文件分块上传
* 文件分块初始化
* @param file
* @param query
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
const { filename, md5, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
async initChunk(query: FileQuery): Promise<void | string> {
const { filename, md5 } = query;
const { absolute, relative } = this.getStoreDir(md5);
const absoluteFilepath = path.join(absolute, filename);
@ -88,6 +84,22 @@ export class LocalOssClient extends BaseOssClient {
return this.serveFilePath(relativeFilePath);
}
return '';
}
/**
* 文件分块上传
* @param file
* @param query
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, chunkIndex } = query;
if (!('chunkIndex' in query)) {
throw new Error('请指定 chunkIndex');
}
const { absolute } = this.getStoreDir(md5);
const chunksDir = path.join(absolute, 'chunks');
fs.ensureDirSync(chunksDir);
fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer);

10
packages/server/src/helpers/file.helper/oss.client.ts
View File

@ -8,7 +8,8 @@ export type FileQuery = {
export abstract class OssClient {
abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string>;
abstract initChunk(query: FileQuery): Promise<void | string>;
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
abstract mergeChunk(query: FileQuery): Promise<string>;
}
@ -25,7 +26,12 @@ export class BaseOssClient implements OssClient {
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
initChunk(query: FileQuery): Promise<void | string> {
throw new Error('Method not implemented.');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
throw new Error('Method not implemented.');
}

158
packages/server/src/helpers/file.helper/tencent.client.ts
View File

@ -1,17 +1,34 @@
import * as TencentCos from 'cos-nodejs-sdk-v5';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as path from 'path';
import { BaseOssClient, FileQuery } from './oss.client';
/**
* 生产环境会以集群方式运行，通过文件来确保 uploadId 只被初始化一次
* @param inOssFileName
* @param uploadId
* @returns
*/
function initUploadId(inOssFileName, uploadId) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
fs.ensureFileSync(uploadIdFile);
return fs.writeFileSync(uploadIdFile, uploadId);
}
function getUploadId(inOssFileName) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
return fs.readFileSync(uploadIdFile, 'utf-8');
}
function deleteUploadId(inOssFileName) {
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
return fs.removeSync(uploadIdFile);
}
export class TencentOssClient extends BaseOssClient {
private client: TencentCos | null;
private uploadIdMap: Map<string, string> = new Map();
private uploadChunkEtagMap: Map<
string,
{
PartNumber: number;
ETag: string;
}[]
> = new Map();
/**
* 构建客户端
@ -109,36 +126,6 @@ export class TencentOssClient extends BaseOssClient {
});
}
/**
* 初始化分块上传
* @param inOssFileName
* @returns
*/
private getUploadChunkId(inOssFileName): Promise<string> {
if (this.uploadIdMap.has(inOssFileName)) {
return Promise.resolve(this.uploadIdMap.get(inOssFileName));
}
return new Promise((resolve, reject) => {
const params = {
Bucket: this.configService.get('oss.tencent.config.Bucket'),
Region: this.configService.get('oss.tencent.config.Region'),
Key: inOssFileName,
};
this.ensureOssClient();
this.client.multipartInit(params, (err, data) => {
if (err) {
reject(err);
} else {
const uploadId = data.UploadId;
this.uploadIdMap.set(inOssFileName, uploadId);
this.uploadChunkEtagMap.set(uploadId, []);
resolve(uploadId);
}
});
});
}
/**
* 上传分片
* @param uploadId
@ -158,14 +145,10 @@ export class TencentOssClient extends BaseOssClient {
Body: file.buffer,
};
this.ensureOssClient();
this.client.multipartUpload(params, (err, data) => {
this.client.multipartUpload(params, (err) => {
if (err) {
reject(err);
} else {
this.uploadChunkEtagMap.get(uploadId).push({
PartNumber: chunkIndex,
ETag: data.ETag,
});
resolve();
}
});
@ -188,26 +171,38 @@ export class TencentOssClient extends BaseOssClient {
Key: inOssFileName,
};
this.ensureOssClient();
const parts = this.uploadChunkEtagMap.get(uploadId);
parts.sort((a, b) => a.PartNumber - b.PartNumber);
this.client.multipartComplete(
this.client.multipartListPart(
{
...params,
UploadId: uploadId,
Parts: parts,
},
(err) => {
(err, data) => {
if (err) {
reject(err);
} else {
this.client.getObjectUrl(params, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data.Url);
const parts = data.Part;
this.client.multipartComplete(
{
...params,
UploadId: uploadId,
Parts: parts,
},
(err) => {
if (err) {
reject(err);
} else {
this.client.getObjectUrl(params, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data.Url);
}
});
}
}
});
);
}
}
);
@ -234,13 +229,52 @@ export class TencentOssClient extends BaseOssClient {
return res as string;
}
/**
* 初始分片
* @param file
* @param query
* @returns
*/
async initChunk(query: FileQuery): Promise<string | void> {
const { md5, filename } = query;
this.ensureOssClient();
const inOssFileName = this.getInOssFileName(md5, filename);
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
if (maybeOssURL) {
return maybeOssURL as string;
}
const params = {
Bucket: this.configService.get('oss.tencent.config.Bucket'),
Region: this.configService.get('oss.tencent.config.Region'),
Key: inOssFileName,
};
const promise = new Promise((resolve, reject) => {
this.client.multipartInit(params, (err, data) => {
if (err) {
reject(err);
} else {
const uploadId = data.UploadId;
initUploadId(inOssFileName, uploadId);
resolve(uploadId);
}
});
});
await promise;
return '';
}
/**
* 上传分片
* @param file
* @param query
* @returns
*/
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
const { md5, filename, chunkIndex } = query;
if (!('chunkIndex' in query)) {
@ -249,15 +283,8 @@ export class TencentOssClient extends BaseOssClient {
this.ensureOssClient();
const inOssFileName = this.getInOssFileName(md5, filename);
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
if (maybeOssURL) {
return maybeOssURL as string;
}
const uploadId = await this.getUploadChunkId(inOssFileName);
const uploadId = getUploadId(inOssFileName);
await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file);
return '';
}
/**
@ -268,10 +295,9 @@ export class TencentOssClient extends BaseOssClient {
async mergeChunk(query: FileQuery): Promise<string> {
const { filename, md5 } = query;
const inOssFileName = this.getInOssFileName(md5, filename);
const uploadId = await this.getUploadChunkId(inOssFileName);
const uploadId = getUploadId(inOssFileName);
const data = await this.completeUploadChunkToCos(uploadId, inOssFileName);
this.uploadIdMap.delete(inOssFileName);
this.uploadChunkEtagMap.delete(uploadId);
deleteUploadId(inOssFileName);
return data;
}
}

4
packages/server/src/services/file.service.ts
View File

@ -14,6 +14,10 @@ export class FileService {
return this.ossClient.uploadFile(file, query);
}
async initChunk(query) {
return this.ossClient.initChunk(query);
}
async uploadChunk(file, query) {
return this.ossClient.uploadChunk(file, query);
}