Merge pull request #65 from fantasticit/fix/file

chore: refactor file serve

packages/client/src/services/file.ts

@@ -83,37 +83,26 @@ export const uploadFile = async (
     const unitPercent = 1 / chunks.length;
     const progressMap = {};
 
-    /**
-     * 先上传一块分块，如果文件已上传，即无需上传后续分块
-     */
-    let url = await uploadFileToServer({
+    let url = await HttpClient.request<string | undefined>({
+      method: FileApiDefinition.initChunk.method,
+      url: FileApiDefinition.initChunk.client(),
+      params: {
         filename,
-      file: chunks[0],
-      chunkIndex: 1,
         md5,
-      isChunk: true,
-      onUploadProgress: (progress) => {
-        progressMap[1] = progress * unitPercent;
-        wraponUploadProgress(
-          Object.keys(progressMap).reduce((a, c) => {
-            return (a += progressMap[c]);
-          }, 0)
-        );
       },
     });
 
     if (!url) {
       await Promise.all(
-        chunks.slice(1).map((chunk, index) => {
-          const currentIndex = 1 + index + 1;
+        chunks.map((chunk, index) => {
           return uploadFileToServer({
             filename,
             file: chunk,
-            chunkIndex: currentIndex,
+            chunkIndex: index + 1,
             md5,
             isChunk: true,
             onUploadProgress: (progress) => {
-              progressMap[currentIndex] = progress * unitPercent;
+              progressMap[index] = progress * unitPercent;
               wraponUploadProgress(
                 Math.min(
                   Object.keys(progressMap).reduce((a, c) => {
@@ -136,9 +125,7 @@ export const uploadFile = async (
         },
       });
     }
-
     wraponUploadProgress(1);
-
     return url;
   }
 };

packages/domains/lib/api/file.d.ts

@@ -7,6 +7,14 @@ export declare const FileApiDefinition: {
         server: "upload";
         client: () => string;
     };
+    /**
+     * 上传分块文件
+     */
+    initChunk: {
+        method: "post";
+        server: "upload/initChunk";
+        client: () => string;
+    };
     /**
      * 上传分块文件
      */

packages/domains/lib/api/file.js

@@ -10,6 +10,14 @@ exports.FileApiDefinition = {
         server: 'upload',
         client: function () { return '/file/upload'; }
     },
+    /**
+     * 上传分块文件
+     */
+    initChunk: {
+        method: 'post',
+        server: 'upload/initChunk',
+        client: function () { return '/file/upload/initChunk'; }
+    },
     /**
      * 上传分块文件
      */

packages/domains/src/api/file.ts

@@ -8,6 +8,15 @@ export const FileApiDefinition = {
     client: () => '/file/upload',
   },
 
+  /**
+   * 初始分块上传
+   */
+  initChunk: {
+    method: 'post' as const,
+    server: 'upload/initChunk' as const,
+    client: () => '/file/upload/initChunk',
+  },
+
   /**
    * 上传分块文件
    */

packages/server/src/controllers/file.controller.ts

@@ -26,6 +26,16 @@ export class FileController {
     return this.fileService.uploadFile(file, query);
   }
 
+  /**
+   * 初始分块文件
+   * @param file
+   */
+  @Post(FileApiDefinition.initChunk.server)
+  @UseGuards(JwtGuard)
+  initChunk(@Query() query: FileQuery) {
+    return this.fileService.initChunk(query);
+  }
+
   /**
    * 上传分块文件
    * @param file

packages/server/src/helpers/file.helper/aliyun.client.ts

@@ -96,18 +96,31 @@ export class AliyunOssClient extends BaseOssClient {
    * @param query
    * @returns
    */
-  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
-    const { md5, filename, chunkIndex } = query;
-
-    if (!('chunkIndex' in query)) {
-      throw new Error('请指定 chunkIndex');
-    }
+  async initChunk(query: FileQuery): Promise<string | void> {
+    const { md5, filename } = query;
 
     const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
     if (maybeOssURL) {
       return maybeOssURL;
     }
 
+    return '';
+  }
+
+  /**
+   * 将切片临时存储到服务器
+   * FIXME: 阿里云的文档没看懂，故做成这种服务器中转的蠢模式
+   * @param file
+   * @param query
+   * @returns
+   */
+  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
+    const { md5, chunkIndex } = query;
+
+    if (!('chunkIndex' in query)) {
+      throw new Error('请指定 chunkIndex');
+    }
+
     const dir = this.getStoreDir(md5);
     const chunksDir = path.join(dir, 'chunks');
     fs.ensureDirSync(chunksDir);

packages/server/src/helpers/file.helper/local.client.ts

@@ -69,16 +69,12 @@ export class LocalOssClient extends BaseOssClient {
   }
 
   /**
-   * 文件分块上传
+   * 文件分块初始化
    * @param file
    * @param query
    */
-  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
-    const { filename, md5, chunkIndex } = query;
-
-    if (!('chunkIndex' in query)) {
-      throw new Error('请指定 chunkIndex');
-    }
+  async initChunk(query: FileQuery): Promise<void | string> {
+    const { filename, md5 } = query;
 
     const { absolute, relative } = this.getStoreDir(md5);
     const absoluteFilepath = path.join(absolute, filename);
@@ -88,6 +84,22 @@ export class LocalOssClient extends BaseOssClient {
       return this.serveFilePath(relativeFilePath);
     }
 
+    return '';
+  }
+
+  /**
+   * 文件分块上传
+   * @param file
+   * @param query
+   */
+  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
+    const { md5, chunkIndex } = query;
+
+    if (!('chunkIndex' in query)) {
+      throw new Error('请指定 chunkIndex');
+    }
+
+    const { absolute } = this.getStoreDir(md5);
     const chunksDir = path.join(absolute, 'chunks');
     fs.ensureDirSync(chunksDir);
     fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer);

packages/server/src/helpers/file.helper/oss.client.ts

@@ -8,7 +8,8 @@ export type FileQuery = {
 
 export abstract class OssClient {
   abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
-  abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string>;
+  abstract initChunk(query: FileQuery): Promise<void | string>;
+  abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
   abstract mergeChunk(query: FileQuery): Promise<string>;
 }
 
@@ -25,7 +26,12 @@ export class BaseOssClient implements OssClient {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void | string> {
+  initChunk(query: FileQuery): Promise<void | string> {
+    throw new Error('Method not implemented.');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
     throw new Error('Method not implemented.');
   }
 

packages/server/src/helpers/file.helper/tencent.client.ts

@@ -1,17 +1,34 @@
 import * as TencentCos from 'cos-nodejs-sdk-v5';
+import * as fs from 'fs-extra';
+import * as os from 'os';
+import * as path from 'path';
 
 import { BaseOssClient, FileQuery } from './oss.client';
 
+/**
+ * 生产环境会以集群方式运行，通过文件来确保 uploadId 只被初始化一次
+ * @param inOssFileName
+ * @param uploadId
+ * @returns
+ */
+function initUploadId(inOssFileName, uploadId) {
+  const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
+  fs.ensureFileSync(uploadIdFile);
+  return fs.writeFileSync(uploadIdFile, uploadId);
+}
+
+function getUploadId(inOssFileName) {
+  const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
+  return fs.readFileSync(uploadIdFile, 'utf-8');
+}
+
+function deleteUploadId(inOssFileName) {
+  const uploadIdFile = path.join(os.tmpdir(), inOssFileName);
+  return fs.removeSync(uploadIdFile);
+}
+
 export class TencentOssClient extends BaseOssClient {
   private client: TencentCos | null;
-  private uploadIdMap: Map<string, string> = new Map();
-  private uploadChunkEtagMap: Map<
-    string,
-    {
-      PartNumber: number;
-      ETag: string;
-    }[]
-  > = new Map();
 
   /**
    * 构建客户端
@@ -109,36 +126,6 @@ export class TencentOssClient extends BaseOssClient {
     });
   }
 
-  /**
-   * 初始化分块上传
-   * @param inOssFileName
-   * @returns
-   */
-  private getUploadChunkId(inOssFileName): Promise<string> {
-    if (this.uploadIdMap.has(inOssFileName)) {
-      return Promise.resolve(this.uploadIdMap.get(inOssFileName));
-    }
-
-    return new Promise((resolve, reject) => {
-      const params = {
-        Bucket: this.configService.get('oss.tencent.config.Bucket'),
-        Region: this.configService.get('oss.tencent.config.Region'),
-        Key: inOssFileName,
-      };
-      this.ensureOssClient();
-      this.client.multipartInit(params, (err, data) => {
-        if (err) {
-          reject(err);
-        } else {
-          const uploadId = data.UploadId;
-          this.uploadIdMap.set(inOssFileName, uploadId);
-          this.uploadChunkEtagMap.set(uploadId, []);
-          resolve(uploadId);
-        }
-      });
-    });
-  }
-
   /**
    * 上传分片
    * @param uploadId
@@ -158,14 +145,10 @@ export class TencentOssClient extends BaseOssClient {
         Body: file.buffer,
       };
       this.ensureOssClient();
-      this.client.multipartUpload(params, (err, data) => {
+      this.client.multipartUpload(params, (err) => {
         if (err) {
           reject(err);
         } else {
-          this.uploadChunkEtagMap.get(uploadId).push({
-            PartNumber: chunkIndex,
-            ETag: data.ETag,
-          });
           resolve();
         }
       });
@@ -188,8 +171,17 @@ export class TencentOssClient extends BaseOssClient {
         Key: inOssFileName,
       };
       this.ensureOssClient();
-      const parts = this.uploadChunkEtagMap.get(uploadId);
-      parts.sort((a, b) => a.PartNumber - b.PartNumber);
+
+      this.client.multipartListPart(
+        {
+          ...params,
+          UploadId: uploadId,
+        },
+        (err, data) => {
+          if (err) {
+            reject(err);
+          } else {
+            const parts = data.Part;
 
             this.client.multipartComplete(
               {
@@ -211,6 +203,9 @@ export class TencentOssClient extends BaseOssClient {
                 }
               }
             );
+          }
+        }
+      );
     });
   }
 
@@ -234,13 +229,52 @@ export class TencentOssClient extends BaseOssClient {
     return res as string;
   }
 
+  /**
+   * 初始分片
+   * @param file
+   * @param query
+   * @returns
+   */
+  async initChunk(query: FileQuery): Promise<string | void> {
+    const { md5, filename } = query;
+    this.ensureOssClient();
+
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
+
+    if (maybeOssURL) {
+      return maybeOssURL as string;
+    }
+
+    const params = {
+      Bucket: this.configService.get('oss.tencent.config.Bucket'),
+      Region: this.configService.get('oss.tencent.config.Region'),
+      Key: inOssFileName,
+    };
+
+    const promise = new Promise((resolve, reject) => {
+      this.client.multipartInit(params, (err, data) => {
+        if (err) {
+          reject(err);
+        } else {
+          const uploadId = data.UploadId;
+          initUploadId(inOssFileName, uploadId);
+          resolve(uploadId);
+        }
+      });
+    });
+
+    await promise;
+    return '';
+  }
+
   /**
    * 上传分片
    * @param file
    * @param query
    * @returns
    */
-  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<string | void> {
+  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
     const { md5, filename, chunkIndex } = query;
 
     if (!('chunkIndex' in query)) {
@@ -249,15 +283,8 @@ export class TencentOssClient extends BaseOssClient {
 
     this.ensureOssClient();
     const inOssFileName = this.getInOssFileName(md5, filename);
-
-    const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName);
-    if (maybeOssURL) {
-      return maybeOssURL as string;
-    }
-
-    const uploadId = await this.getUploadChunkId(inOssFileName);
+    const uploadId = getUploadId(inOssFileName);
     await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file);
-    return '';
   }
 
   /**
@@ -268,10 +295,9 @@ export class TencentOssClient extends BaseOssClient {
   async mergeChunk(query: FileQuery): Promise<string> {
     const { filename, md5 } = query;
     const inOssFileName = this.getInOssFileName(md5, filename);
-    const uploadId = await this.getUploadChunkId(inOssFileName);
+    const uploadId = getUploadId(inOssFileName);
     const data = await this.completeUploadChunkToCos(uploadId, inOssFileName);
-    this.uploadIdMap.delete(inOssFileName);
-    this.uploadChunkEtagMap.delete(uploadId);
+    deleteUploadId(inOssFileName);
     return data;
   }
 }

packages/server/src/services/file.service.ts

@@ -14,6 +14,10 @@ export class FileService {
     return this.ossClient.uploadFile(file, query);
   }
 
+  async initChunk(query) {
+    return this.ossClient.initChunk(query);
+  }
+
   async uploadChunk(file, query) {
     return this.ossClient.uploadChunk(file, query);
   }