mirror of https://github.com/fantasticit/think.git
Merge pull request #65 from fantasticit/fix/file
chore: refactor file serve
This commit is contained in:
commit
849c6ab91f
|
@ -83,37 +83,26 @@ export const uploadFile = async (
|
||||||
const unitPercent = 1 / chunks.length;
|
const unitPercent = 1 / chunks.length;
|
||||||
const progressMap = {};
|
const progressMap = {};
|
||||||
|
|
||||||
/**
|
let url = await HttpClient.request<string | undefined>({
|
||||||
* 先上传一块分块,如果文件已上传,即无需上传后续分块
|
method: FileApiDefinition.initChunk.method,
|
||||||
*/
|
url: FileApiDefinition.initChunk.client(),
|
||||||
let url = await uploadFileToServer({
|
params: {
|
||||||
filename,
|
filename,
|
||||||
file: chunks[0],
|
|
||||||
chunkIndex: 1,
|
|
||||||
md5,
|
md5,
|
||||||
isChunk: true,
|
|
||||||
onUploadProgress: (progress) => {
|
|
||||||
progressMap[1] = progress * unitPercent;
|
|
||||||
wraponUploadProgress(
|
|
||||||
Object.keys(progressMap).reduce((a, c) => {
|
|
||||||
return (a += progressMap[c]);
|
|
||||||
}, 0)
|
|
||||||
);
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!url) {
|
if (!url) {
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
chunks.slice(1).map((chunk, index) => {
|
chunks.map((chunk, index) => {
|
||||||
const currentIndex = 1 + index + 1;
|
|
||||||
return uploadFileToServer({
|
return uploadFileToServer({
|
||||||
filename,
|
filename,
|
||||||
file: chunk,
|
file: chunk,
|
||||||
chunkIndex: currentIndex,
|
chunkIndex: index + 1,
|
||||||
md5,
|
md5,
|
||||||
isChunk: true,
|
isChunk: true,
|
||||||
onUploadProgress: (progress) => {
|
onUploadProgress: (progress) => {
|
||||||
progressMap[currentIndex] = progress * unitPercent;
|
progressMap[index] = progress * unitPercent;
|
||||||
wraponUploadProgress(
|
wraponUploadProgress(
|
||||||
Math.min(
|
Math.min(
|
||||||
Object.keys(progressMap).reduce((a, c) => {
|
Object.keys(progressMap).reduce((a, c) => {
|
||||||
|
@ -136,9 +125,7 @@ export const uploadFile = async (
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
wraponUploadProgress(1);
|
wraponUploadProgress(1);
|
||||||
|
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -7,6 +7,14 @@ export declare const FileApiDefinition: {
|
||||||
server: "upload";
|
server: "upload";
|
||||||
client: () => string;
|
client: () => string;
|
||||||
};
|
};
|
||||||
|
/**
|
||||||
|
* 上传分块文件
|
||||||
|
*/
|
||||||
|
initChunk: {
|
||||||
|
method: "post";
|
||||||
|
server: "upload/initChunk";
|
||||||
|
client: () => string;
|
||||||
|
};
|
||||||
/**
|
/**
|
||||||
* 上传分块文件
|
* 上传分块文件
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -10,6 +10,14 @@ exports.FileApiDefinition = {
|
||||||
server: 'upload',
|
server: 'upload',
|
||||||
client: function () { return '/file/upload'; }
|
client: function () { return '/file/upload'; }
|
||||||
},
|
},
|
||||||
|
/**
|
||||||
|
* 上传分块文件
|
||||||
|
*/
|
||||||
|
initChunk: {
|
||||||
|
method: 'post',
|
||||||
|
server: 'upload/initChunk',
|
||||||
|
client: function () { return '/file/upload/initChunk'; }
|
||||||
|
},
|
||||||
/**
|
/**
|
||||||
* 上传分块文件
|
* 上传分块文件
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -8,6 +8,15 @@ export const FileApiDefinition = {
|
||||||
client: () => '/file/upload',
|
client: () => '/file/upload',
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 初始分块上传
|
||||||
|
*/
|
||||||
|
initChunk: {
|
||||||
|
method: 'post' as const,
|
||||||
|
server: 'upload/initChunk' as const,
|
||||||
|
client: () => '/file/upload/initChunk',
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传分块文件
|
* 上传分块文件
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -26,6 +26,16 @@ export class FileController {
|
||||||
return this.fileService.uploadFile(file, query);
|
return this.fileService.uploadFile(file, query);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 初始分块文件
|
||||||
|
* @param file
|
||||||
|
*/
|
||||||
|
@Post(FileApiDefinition.initChunk.server)
|
||||||
|
@UseGuards(JwtGuard)
|
||||||
|
initChunk(@Query() query: FileQuery) {
|
||||||
|
return this.fileService.initChunk(query);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传分块文件
|
* 上传分块文件
|
||||||
* @param file
|
* @param file
|
||||||
|
|
|
@ -96,18 +96,31 @@ export class AliyunOssClient extends BaseOssClient {
|
||||||
* @param query
|
* @param query
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
|
async initChunk(query: FileQuery): Promise<string | void> {
|
||||||
const { md5, filename, chunkIndex } = query;
|
const { md5, filename } = query;
|
||||||
|
|
||||||
if (!('chunkIndex' in query)) {
|
|
||||||
throw new Error('请指定 chunkIndex');
|
|
||||||
}
|
|
||||||
|
|
||||||
const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
|
const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
|
||||||
if (maybeOssURL) {
|
if (maybeOssURL) {
|
||||||
return maybeOssURL;
|
return maybeOssURL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将切片临时存储到服务器
|
||||||
|
* FIXME: 阿里云的文档没看懂,故做成这种服务器中转的蠢模式
|
||||||
|
* @param file
|
||||||
|
* @param query
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
|
||||||
|
const { md5, chunkIndex } = query;
|
||||||
|
|
||||||
|
if (!('chunkIndex' in query)) {
|
||||||
|
throw new Error('请指定 chunkIndex');
|
||||||
|
}
|
||||||
|
|
||||||
const dir = this.getStoreDir(md5);
|
const dir = this.getStoreDir(md5);
|
||||||
const chunksDir = path.join(dir, 'chunks');
|
const chunksDir = path.join(dir, 'chunks');
|
||||||
fs.ensureDirSync(chunksDir);
|
fs.ensureDirSync(chunksDir);
|
||||||
|
|
|
@ -69,16 +69,12 @@ export class LocalOssClient extends BaseOssClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 文件分块上传
|
* 文件分块初始化
|
||||||
* @param file
|
* @param file
|
||||||
* @param query
|
* @param query
|
||||||
*/
|
*/
|
||||||
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
|
async initChunk(query: FileQuery): Promise<void | string> {
|
||||||
const { filename, md5, chunkIndex } = query;
|
const { filename, md5 } = query;
|
||||||
|
|
||||||
if (!('chunkIndex' in query)) {
|
|
||||||
throw new Error('请指定 chunkIndex');
|
|
||||||
}
|
|
||||||
|
|
||||||
const { absolute, relative } = this.getStoreDir(md5);
|
const { absolute, relative } = this.getStoreDir(md5);
|
||||||
const absoluteFilepath = path.join(absolute, filename);
|
const absoluteFilepath = path.join(absolute, filename);
|
||||||
|
@ -88,6 +84,22 @@ export class LocalOssClient extends BaseOssClient {
|
||||||
return this.serveFilePath(relativeFilePath);
|
return this.serveFilePath(relativeFilePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文件分块上传
|
||||||
|
* @param file
|
||||||
|
* @param query
|
||||||
|
*/
|
||||||
|
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
|
||||||
|
const { md5, chunkIndex } = query;
|
||||||
|
|
||||||
|
if (!('chunkIndex' in query)) {
|
||||||
|
throw new Error('请指定 chunkIndex');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { absolute } = this.getStoreDir(md5);
|
||||||
const chunksDir = path.join(absolute, 'chunks');
|
const chunksDir = path.join(absolute, 'chunks');
|
||||||
fs.ensureDirSync(chunksDir);
|
fs.ensureDirSync(chunksDir);
|
||||||
fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer);
|
fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer);
|
||||||
|
|
|
@ -8,7 +8,8 @@ export type FileQuery = {
|
||||||
|
|
||||||
export abstract class OssClient {
|
export abstract class OssClient {
|
||||||
abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
|
abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
|
||||||
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string>;
|
abstract initChunk(query: FileQuery): Promise<void | string>;
|
||||||
|
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
|
||||||
abstract mergeChunk(query: FileQuery): Promise<string>;
|
abstract mergeChunk(query: FileQuery): Promise<string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -25,7 +26,12 @@ export class BaseOssClient implements OssClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
|
initChunk(query: FileQuery): Promise<void | string> {
|
||||||
|
throw new Error('Method not implemented.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
|
uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
|
||||||
throw new Error('Method not implemented.');
|
throw new Error('Method not implemented.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,17 +1,34 @@
|
||||||
import * as TencentCos from 'cos-nodejs-sdk-v5';
|
import * as TencentCos from 'cos-nodejs-sdk-v5';
|
||||||
|
import * as fs from 'fs-extra';
|
||||||
|
import * as os from 'os';
|
||||||
|
import * as path from 'path';
|
||||||
|
|
||||||
import { BaseOssClient, FileQuery } from './oss.client';
|
import { BaseOssClient, FileQuery } from './oss.client';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生产环境会以集群方式运行,通过文件来确保 uploadId 只被初始化一次
|
||||||
|
* @param inOssFileName
|
||||||
|
* @param uploadId
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
function initUploadId(inOssFileName, uploadId) {
|
||||||
|
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
|
||||||
|
fs.ensureFileSync(uploadIdFile);
|
||||||
|
return fs.writeFileSync(uploadIdFile, uploadId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getUploadId(inOssFileName) {
|
||||||
|
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
|
||||||
|
return fs.readFileSync(uploadIdFile, 'utf-8');
|
||||||
|
}
|
||||||
|
|
||||||
|
function deleteUploadId(inOssFileName) {
|
||||||
|
const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
|
||||||
|
return fs.removeSync(uploadIdFile);
|
||||||
|
}
|
||||||
|
|
||||||
export class TencentOssClient extends BaseOssClient {
|
export class TencentOssClient extends BaseOssClient {
|
||||||
private client: TencentCos | null;
|
private client: TencentCos | null;
|
||||||
private uploadIdMap: Map<string, string> = new Map();
|
|
||||||
private uploadChunkEtagMap: Map<
|
|
||||||
string,
|
|
||||||
{
|
|
||||||
PartNumber: number;
|
|
||||||
ETag: string;
|
|
||||||
}[]
|
|
||||||
> = new Map();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 构建客户端
|
* 构建客户端
|
||||||
|
@ -109,36 +126,6 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 初始化分块上传
|
|
||||||
* @param inOssFileName
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
private getUploadChunkId(inOssFileName): Promise<string> {
|
|
||||||
if (this.uploadIdMap.has(inOssFileName)) {
|
|
||||||
return Promise.resolve(this.uploadIdMap.get(inOssFileName));
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const params = {
|
|
||||||
Bucket: this.configService.get('oss.tencent.config.Bucket'),
|
|
||||||
Region: this.configService.get('oss.tencent.config.Region'),
|
|
||||||
Key: inOssFileName,
|
|
||||||
};
|
|
||||||
this.ensureOssClient();
|
|
||||||
this.client.multipartInit(params, (err, data) => {
|
|
||||||
if (err) {
|
|
||||||
reject(err);
|
|
||||||
} else {
|
|
||||||
const uploadId = data.UploadId;
|
|
||||||
this.uploadIdMap.set(inOssFileName, uploadId);
|
|
||||||
this.uploadChunkEtagMap.set(uploadId, []);
|
|
||||||
resolve(uploadId);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传分片
|
* 上传分片
|
||||||
* @param uploadId
|
* @param uploadId
|
||||||
|
@ -158,14 +145,10 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
Body: file.buffer,
|
Body: file.buffer,
|
||||||
};
|
};
|
||||||
this.ensureOssClient();
|
this.ensureOssClient();
|
||||||
this.client.multipartUpload(params, (err, data) => {
|
this.client.multipartUpload(params, (err) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
reject(err);
|
reject(err);
|
||||||
} else {
|
} else {
|
||||||
this.uploadChunkEtagMap.get(uploadId).push({
|
|
||||||
PartNumber: chunkIndex,
|
|
||||||
ETag: data.ETag,
|
|
||||||
});
|
|
||||||
resolve();
|
resolve();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -188,8 +171,17 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
Key: inOssFileName,
|
Key: inOssFileName,
|
||||||
};
|
};
|
||||||
this.ensureOssClient();
|
this.ensureOssClient();
|
||||||
const parts = this.uploadChunkEtagMap.get(uploadId);
|
|
||||||
parts.sort((a, b) => a.PartNumber - b.PartNumber);
|
this.client.multipartListPart(
|
||||||
|
{
|
||||||
|
...params,
|
||||||
|
UploadId: uploadId,
|
||||||
|
},
|
||||||
|
(err, data) => {
|
||||||
|
if (err) {
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
const parts = data.Part;
|
||||||
|
|
||||||
this.client.multipartComplete(
|
this.client.multipartComplete(
|
||||||
{
|
{
|
||||||
|
@ -211,6 +203,9 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -234,13 +229,52 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
return res as string;
|
return res as string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 初始分片
|
||||||
|
* @param file
|
||||||
|
* @param query
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
async initChunk(query: FileQuery): Promise<string | void> {
|
||||||
|
const { md5, filename } = query;
|
||||||
|
this.ensureOssClient();
|
||||||
|
|
||||||
|
const inOssFileName = this.getInOssFileName(md5, filename);
|
||||||
|
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
|
||||||
|
|
||||||
|
if (maybeOssURL) {
|
||||||
|
return maybeOssURL as string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const params = {
|
||||||
|
Bucket: this.configService.get('oss.tencent.config.Bucket'),
|
||||||
|
Region: this.configService.get('oss.tencent.config.Region'),
|
||||||
|
Key: inOssFileName,
|
||||||
|
};
|
||||||
|
|
||||||
|
const promise = new Promise((resolve, reject) => {
|
||||||
|
this.client.multipartInit(params, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
const uploadId = data.UploadId;
|
||||||
|
initUploadId(inOssFileName, uploadId);
|
||||||
|
resolve(uploadId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await promise;
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传分片
|
* 上传分片
|
||||||
* @param file
|
* @param file
|
||||||
* @param query
|
* @param query
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
|
async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
|
||||||
const { md5, filename, chunkIndex } = query;
|
const { md5, filename, chunkIndex } = query;
|
||||||
|
|
||||||
if (!('chunkIndex' in query)) {
|
if (!('chunkIndex' in query)) {
|
||||||
|
@ -249,15 +283,8 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
|
|
||||||
this.ensureOssClient();
|
this.ensureOssClient();
|
||||||
const inOssFileName = this.getInOssFileName(md5, filename);
|
const inOssFileName = this.getInOssFileName(md5, filename);
|
||||||
|
const uploadId = getUploadId(inOssFileName);
|
||||||
const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
|
|
||||||
if (maybeOssURL) {
|
|
||||||
return maybeOssURL as string;
|
|
||||||
}
|
|
||||||
|
|
||||||
const uploadId = await this.getUploadChunkId(inOssFileName);
|
|
||||||
await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file);
|
await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file);
|
||||||
return '';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -268,10 +295,9 @@ export class TencentOssClient extends BaseOssClient {
|
||||||
async mergeChunk(query: FileQuery): Promise<string> {
|
async mergeChunk(query: FileQuery): Promise<string> {
|
||||||
const { filename, md5 } = query;
|
const { filename, md5 } = query;
|
||||||
const inOssFileName = this.getInOssFileName(md5, filename);
|
const inOssFileName = this.getInOssFileName(md5, filename);
|
||||||
const uploadId = await this.getUploadChunkId(inOssFileName);
|
const uploadId = getUploadId(inOssFileName);
|
||||||
const data = await this.completeUploadChunkToCos(uploadId, inOssFileName);
|
const data = await this.completeUploadChunkToCos(uploadId, inOssFileName);
|
||||||
this.uploadIdMap.delete(inOssFileName);
|
deleteUploadId(inOssFileName);
|
||||||
this.uploadChunkEtagMap.delete(uploadId);
|
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,10 @@ export class FileService {
|
||||||
return this.ossClient.uploadFile(file, query);
|
return this.ossClient.uploadFile(file, query);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async initChunk(query) {
|
||||||
|
return this.ossClient.initChunk(query);
|
||||||
|
}
|
||||||
|
|
||||||
async uploadChunk(file, query) {
|
async uploadChunk(file, query) {
|
||||||
return this.ossClient.uploadChunk(file, query);
|
return this.ossClient.uploadChunk(file, query);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue