diff --git a/config/dev.yaml b/config/dev.yaml index 80e0292e..13efc0fc 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -73,4 +73,4 @@ oss: # jwt 配置 jwt: secretkey: 'zA_Think+KNOWLEDGE+WIKI+DOCUMENTS@2022' - expiresIn: '6h' + expiresIn: '6h' \ No newline at end of file diff --git a/packages/client/next.config.js b/packages/client/next.config.js index 5c4bedcf..e93f21a5 100644 --- a/packages/client/next.config.js +++ b/packages/client/next.config.js @@ -23,7 +23,7 @@ const nextConfig = semi({ env: { SERVER_API_URL: config.client.apiUrl, COLLABORATION_API_URL: config.client.collaborationUrl, - ENABLE_ALIYUN_OSS: !!config.oss.aliyun.accessKeyId, + ENABLE_OSS_S3: config.oss.s3.enable, DNS_PREFETCH: (config.client.dnsPrefetch || '').split(' '), SEO_APPNAME: config.client.seoAppName, SEO_DESCRIPTION: config.client.seoDescription, diff --git a/packages/client/src/components/resizeable/resizeable.tsx b/packages/client/src/components/resizeable/resizeable.tsx index 71db4fe4..e8e82c2b 100644 --- a/packages/client/src/components/resizeable/resizeable.tsx +++ b/packages/client/src/components/resizeable/resizeable.tsx @@ -9,8 +9,8 @@ import styles from './style.module.scss'; type ISize = { width: number; height: number }; interface IProps { - width: number; - height: number; + width: number | string; + height: number | string; maxWidth?: number; isEditable?: boolean; onChange?: (arg: ISize) => void; diff --git a/packages/client/src/services/file.ts b/packages/client/src/services/file.ts index 457b1529..af778030 100644 --- a/packages/client/src/services/file.ts +++ b/packages/client/src/services/file.ts @@ -1,6 +1,11 @@ +import { Toast } from '@douyinfe/semi-ui'; + import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains'; +import axios from 'axios'; +import { url } from 'inspector'; import { string } from 'lib0'; +import { timeout } from 'lib0/eventloop'; import SparkMD5 from 'spark-md5'; import { HttpClient } from './http-client'; @@ -35,7 +40,6 @@ const uploadFileToServer = (arg: { }) => { const { filename, file, md5, isChunk, chunkIndex, onUploadProgress } = arg; const api = isChunk ? 'uploadChunk' : 'upload'; - const formData = new FormData(); formData.append('file', file); @@ -51,6 +55,7 @@ const uploadFileToServer = (arg: { md5, chunkIndex, }, + timeout: 30 * 1000, onUploadProgress: (progress) => { const percent = progress.loaded / progress.total; onUploadProgress && onUploadProgress(percent); @@ -67,68 +72,152 @@ export const uploadFile = async ( return onUploadProgress && onUploadProgress(Math.ceil(percent * 100)); }; - const filename = file.name; + // 开启s3 文件上传支持 + if (!process.env.ENABLE_OSS_S3) { + const filename = file.name; + console.debug('当前没有开启oss 对象存储,使用本地上传方案'); + if (file.size > FILE_CHUNK_SIZE) { + onTooLarge && onTooLarge(); + } - if (file.size > FILE_CHUNK_SIZE * 5) { - onTooLarge && onTooLarge(); - } + if (file.size <= FILE_CHUNK_SIZE) { + const spark = new SparkMD5(); + spark.append(file); + spark.append(file.lastModified); + spark.append(file.type); + const md5 = spark.end(); + const url = await uploadFileToServer({ filename, file, md5, onUploadProgress: wraponUploadProgress }); + return url; + } else { + const { chunks, md5 } = await splitBigFile(file); + const unitPercent = 1 / chunks.length; + const progressMap = {}; - if (file.size <= FILE_CHUNK_SIZE) { - const spark = new SparkMD5(); - spark.append(file); - spark.append(file.lastModified); - spark.append(file.type); - const md5 = spark.end(); - const url = await uploadFileToServer({ filename, file, md5, onUploadProgress: wraponUploadProgress }); - return url; - } else { - const { chunks, md5 } = await splitBigFile(file); - const unitPercent = 1 / chunks.length; - const progressMap = {}; - - let url = await HttpClient.request({ - method: FileApiDefinition.initChunk.method, - url: FileApiDefinition.initChunk.client(), - params: { - filename, - md5, - }, - }); - - if (!url) { - await Promise.all( - chunks.map((chunk, index) => { - return uploadFileToServer({ - filename, - file: chunk, - chunkIndex: index + 1, - md5, - isChunk: true, - onUploadProgress: (progress) => { - progressMap[index] = progress * unitPercent; - wraponUploadProgress( - Math.min( - Object.keys(progressMap).reduce((a, c) => { - return (a += progressMap[c]); - }, 0), - // 剩下的 5% 交给 merge - 0.95 - ) - ); - }, - }); - }) - ); - url = await HttpClient.request({ - method: FileApiDefinition.mergeChunk.method, - url: FileApiDefinition.mergeChunk.client(), + let url = await HttpClient.request({ + method: FileApiDefinition.initChunk.method, + url: FileApiDefinition.initChunk.client(), params: { filename, md5, }, }); + + if (!url) { + await Promise.all( + chunks.map((chunk, index) => { + return uploadFileToServer({ + filename, + file: chunk, + chunkIndex: index + 1, + md5, + isChunk: true, + onUploadProgress: (progress) => { + progressMap[index] = progress * unitPercent; + wraponUploadProgress( + Math.min( + Object.keys(progressMap).reduce((a, c) => { + return (a += progressMap[c]); + }, 0), + // 剩下的 5% 交给 merge + 0.95 + ) + ); + }, + }); + }) + ); + url = await HttpClient.request({ + method: FileApiDefinition.mergeChunk.method, + url: FileApiDefinition.mergeChunk.client(), + params: { + filename, + md5, + }, + }); + } + wraponUploadProgress(1); + return url; + } + } + // S3 后端签名 前端文件直传 方案 + else { + // 前端计算文件的md5 + console.log('计算待上传的文件{' + file.name + '}的md5...'); + const { chunks, md5 } = await splitBigFile(file); + console.log('文件{' + file.name + '}的md5:' + md5); + const filename = file.name; + + // 请求后端检查指定的文件是不是已经存在 + const res = await HttpClient.request({ + method: FileApiDefinition.ossSign.method, + url: FileApiDefinition.ossSign.client(), + data: { filename, md5, fileSize: file.size }, + }); + // 如果后端反应文件已经存在 + if (res['isExist']) { + Toast.info('文件秒传成功!'); + return res['objectUrl']; + } else { + //console.log('文件不存在,需要上传文件'); + // 后端认为文件小,前端直接put 上传 + if (!res['MultipartUpload']) { + console.log('前端直接PUT上传文件'); + const signUrl = res['signUrl']; + await axios.put(signUrl, file, { + timeout: 120 * 1000, + onUploadProgress: (process) => { + const uploadLoaded = process.loaded; + const uploadTotal = file.size; + const uploadPercent = uploadLoaded / uploadTotal; + wraponUploadProgress(uploadPercent); + }, + }); + const upres = await HttpClient.request({ + method: FileApiDefinition.ossSign.method, + url: FileApiDefinition.ossSign.client(), + data: { filename, md5, fileSize: file.size }, + }); + return upres['objectUrl']; + } + // 前端进入分片上传流程 + else { + const upload_id = res['uploadId']; + // console.log('分片文件上传,upload_id:' + upload_id); + const MultipartUpload = []; + for (let index = 0; index < chunks.length; index++) { + const chunk = chunks[index]; + const res = await HttpClient.request({ + method: FileApiDefinition.ossChunk.method, + url: FileApiDefinition.ossChunk.client(), + data: { filename, md5, uploadId: upload_id, chunkIndex: index + 1 }, + }); + // 上传文件分块到s3 + // 直接用原生请求不走拦截器 + const upload_res = await axios.put(res['signUrl'], chunk, { + timeout: 120 * 1000, + onUploadProgress: (process) => { + const uploadLoaded = process.loaded + FILE_CHUNK_SIZE * index; + const uploadTotal = file.size; + const uploadPercent = uploadLoaded / uploadTotal; + //console.log(uploadLoaded, uploadTotal, uploadPercent); + wraponUploadProgress(uploadPercent); + }, + }); + const upload_etag = upload_res.headers['etag']; + const response_part = { PartNumber: index + 1, ETag: upload_etag }; + MultipartUpload.push(response_part); + //console.log('文件分片{' + (index + 1) + '上传成功,etag:' + upload_etag); + } + // 文件已经全部上传OK + // 请求后端合并文件 + const payload = { filename, md5, uploadId: upload_id, MultipartUpload }; + const upres = await HttpClient.request({ + method: FileApiDefinition.ossMerge.method, + url: FileApiDefinition.ossMerge.client(), + data: payload, + }); + return '' + upres; + } } - wraponUploadProgress(1); - return url; } }; diff --git a/packages/client/src/services/http-client.ts b/packages/client/src/services/http-client.ts index 4516fecc..d35c39b5 100644 --- a/packages/client/src/services/http-client.ts +++ b/packages/client/src/services/http-client.ts @@ -39,6 +39,7 @@ HttpClient.interceptors.response.use( isBrowser && Toast.error(data.data.message); return null; } + // 如果是 204 请求 那么直接返回 data.headers const res = data.data; diff --git a/packages/client/src/tiptap/core/wrappers/image/index.tsx b/packages/client/src/tiptap/core/wrappers/image/index.tsx index e64e4c3e..98cc90b0 100644 --- a/packages/client/src/tiptap/core/wrappers/image/index.tsx +++ b/packages/client/src/tiptap/core/wrappers/image/index.tsx @@ -88,9 +88,8 @@ export const ImageWrapper = ({ editor, node, updateAttributes }) => { @@ -106,11 +105,9 @@ export const ImageWrapper = ({ editor, node, updateAttributes }) => { ) : ( -
+
{ transition: `all ease-in-out .3s`, }} > - +
diff --git a/packages/client/src/tiptap/prose-utils/download.ts b/packages/client/src/tiptap/prose-utils/download.ts index 75a3c491..72655100 100644 --- a/packages/client/src/tiptap/prose-utils/download.ts +++ b/packages/client/src/tiptap/prose-utils/download.ts @@ -1,5 +1,6 @@ import FileSaver from 'file-saver'; export function download(url, name) { - FileSaver.saveAs(url, name); + if (url.startsWith('http://') || url.startsWith('https://')) window.open(url, '文件下载...'); + else FileSaver.saveAs(url, name); } diff --git a/packages/domains/lib/api/file.d.ts b/packages/domains/lib/api/file.d.ts index 72e14e3e..67d805d7 100644 --- a/packages/domains/lib/api/file.d.ts +++ b/packages/domains/lib/api/file.d.ts @@ -1,3 +1,4 @@ + export declare const FileApiDefinition: { /** * 上传文件 @@ -31,5 +32,33 @@ export declare const FileApiDefinition: { server: "merge/chunk"; client: () => string; }; + + /** + * 后端签名生成需要上传的文件 + */ + ossSign:{ + method: "post"; + server: "upload/ossSign"; + client: () => string; + }; + + /** + * 后端签名上传分片 + */ + ossChunk:{ + method: "post"; + server: "upload/ossChunk"; + client: () => string; + }; + + /** + * 后端签名上传结束 + */ + ossMerge:{ + method: "post"; + server: "upload/ossMerge"; + client: () => string; + }; + }; export declare const FILE_CHUNK_SIZE: number; diff --git a/packages/domains/lib/api/file.js b/packages/domains/lib/api/file.js index 0ae6eba3..a9b7d062 100644 --- a/packages/domains/lib/api/file.js +++ b/packages/domains/lib/api/file.js @@ -33,7 +33,22 @@ exports.FileApiDefinition = { method: 'post', server: 'merge/chunk', client: function () { return '/file/merge/chunk'; } - } + }, + ossSign:{ + method: "post", + server: "upload/ossSign", + client: function () { return '/file/upload/ossSign'; } + }, + ossChunk:{ + method: "post", + server: "upload/ossChunk", + client: function () { return '/file/upload/ossChunk'; } + }, + ossMerge:{ + method: "post", + server: "upload/ossMerge", + client: function () { return '/file/upload/ossMerge'; } + }, }; // 设置文件分片的大小 改成 8 M // MINIO 等oss 有最小分片的限制 diff --git a/packages/server/src/controllers/file.controller.ts b/packages/server/src/controllers/file.controller.ts index c9067acb..3ae44e8b 100644 --- a/packages/server/src/controllers/file.controller.ts +++ b/packages/server/src/controllers/file.controller.ts @@ -1,10 +1,10 @@ -import { Controller, Post, Query, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common'; +import { Body, Controller, Post, Query, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains'; import { JwtGuard } from '@guard/jwt.guard'; -import { FileQuery } from '@helpers/file.helper/oss.client'; +import { FileMerge, FileQuery } from '@helpers/file.helper/oss.client'; import { FileService } from '@services/file.service'; @Controller('file') @@ -64,4 +64,31 @@ export class FileController { mergeChunk(@Query() query: FileQuery) { return this.fileService.mergeChunk(query); } + + /** + * 请求后端签名前端直传 + */ + @Post(FileApiDefinition.ossSign.server) + @UseGuards(JwtGuard) + ossSign(@Body() data: FileQuery) { + return this.fileService.ossSign(data); + } + + /** + * 请求后端对分片上传的文件进行签名 + */ + @Post(FileApiDefinition.ossChunk.server) + @UseGuards(JwtGuard) + ossChunk(@Body() data: FileQuery) { + return this.fileService.ossChunk(data); + } + + /** + * 请求后端合并分片上传的文件 + */ + @Post(FileApiDefinition.ossMerge.server) + @UseGuards(JwtGuard) + ossMerge(@Body() data: FileMerge) { + return this.fileService.ossMerge(data); + } } diff --git a/packages/server/src/helpers/file.helper/oss.client.ts b/packages/server/src/helpers/file.helper/oss.client.ts index 5dba108c..dc0c2d15 100644 --- a/packages/server/src/helpers/file.helper/oss.client.ts +++ b/packages/server/src/helpers/file.helper/oss.client.ts @@ -1,11 +1,42 @@ import { ConfigService } from '@nestjs/config'; +import exp from 'constants'; import Redis from 'ioredis'; export type FileQuery = { filename: string; md5: string; chunkIndex?: number; + fileSize?: number; + uploadId?: string; +}; + +export type FileMerge = { + filename: string; + md5: string; + uploadId: string; + MultipartUpload: any; +}; + +export type chunkUpload = { + uploadId: string; + chunkIndex: number; + etag: string; +}; + +export type ossSignReponse = { + MultipartUpload: boolean; + isExist: boolean; + uploadId: string | null; + objectKey: string; + objectUrl: string | null; + signUrl: string | null; +}; + +export type ossChunkResponse = { + signUrl: string; + uploadId: string; + chunkIndex: number; }; export abstract class OssClient { @@ -14,6 +45,9 @@ export abstract class OssClient { abstract initChunk(query: FileQuery): Promise; abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise; abstract mergeChunk(query: FileQuery): Promise; + abstract ossSign(query: FileQuery): Promise; + abstract ossChunk(query: FileQuery): Promise; + abstract ossMerge(query: FileMerge): Promise; } export class BaseOssClient implements OssClient { @@ -43,6 +77,21 @@ export class BaseOssClient implements OssClient { throw new Error('Method not implemented.'); } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ossSign(query: FileQuery): Promise { + throw new Error('Method not implemented.'); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ossMerge(query: FileMerge): Promise { + throw new Error('Method not implemented.'); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ossChunk(query: FileQuery): Promise { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars setRedis(redis: Redis): Promise { throw new Error('Method not implemented.'); diff --git a/packages/server/src/helpers/file.helper/s3.client.ts b/packages/server/src/helpers/file.helper/s3.client.ts index 1d34577b..def5688b 100644 --- a/packages/server/src/helpers/file.helper/s3.client.ts +++ b/packages/server/src/helpers/file.helper/s3.client.ts @@ -1,3 +1,5 @@ +import { FILE_CHUNK_SIZE } from '@think/domains'; + import { CompleteMultipartUploadCommand, CreateMultipartUploadCommand, @@ -10,7 +12,7 @@ import { import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import Redis from 'ioredis'; -import { BaseOssClient, FileQuery } from './oss.client'; +import { BaseOssClient, FileMerge, FileQuery, ossChunkResponse, ossSignReponse } from './oss.client'; export class S3OssClient extends BaseOssClient { private client: S3Client | null; @@ -103,7 +105,6 @@ export class S3OssClient extends BaseOssClient { this.ensureS3OssClient(); const command = new GetObjectCommand({ Bucket: bucket, Key: key }); const signUrl = await getSignedUrl(this.client, command); - console.log('signUrl:' + signUrl); return signUrl.split('?')[0]; } @@ -221,7 +222,6 @@ export class S3OssClient extends BaseOssClient { const obj = JSON.parse(await this.redis.get('think:oss:chunk:' + md5 + ':' + i)); MultipartUpload.Parts.push(obj); } - console.log(MultipartUpload, upload_id); const command = new CompleteMultipartUploadCommand({ Bucket: this.bucket, Key: inOssFileName, @@ -234,4 +234,73 @@ export class S3OssClient extends BaseOssClient { await this.redis.del('think:oss:chunk:' + md5 + '*'); return await this.getObjectUrl(this.bucket, inOssFileName); } + + async ossSign(query: FileQuery): Promise { + const { filename, md5, fileSize } = query; + const inOssFileName = await this.getInOssFileName(md5, filename); + this.ensureS3OssClient(); + const objectUrl = await this.checkIfAlreadyInOss(md5, filename); + if (objectUrl) { + return { + signUrl: null, + MultipartUpload: false, + uploadId: null, + objectKey: inOssFileName, + isExist: true, + objectUrl: objectUrl, + }; + } + if (fileSize <= FILE_CHUNK_SIZE) { + const command = new PutObjectCommand({ Bucket: this.bucket, Key: inOssFileName }); + const signUrl = await getSignedUrl(this.client, command); + return { + signUrl: signUrl, + MultipartUpload: false, + uploadId: null, + objectKey: inOssFileName, + isExist: false, + objectUrl: null, + }; + } else { + const command = new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: inOssFileName }); + const response = await this.client.send(command); + const upload_id = response['UploadId']; + return { + signUrl: null, + MultipartUpload: true, + uploadId: upload_id, + objectKey: inOssFileName, + isExist: false, + objectUrl: null, + }; + } + } + + async ossChunk(query: FileQuery): Promise { + this.ensureS3OssClient(); + const { filename, md5 } = query; + const inOssFileName = await this.getInOssFileName(md5, filename); + const command = new UploadPartCommand({ + UploadId: query.uploadId, + Bucket: this.bucket, + Key: inOssFileName, + PartNumber: query.chunkIndex, + }); + const signUrl = await getSignedUrl(this.client, command); + return { signUrl: signUrl, uploadId: query.uploadId, chunkIndex: query.chunkIndex }; + } + + async ossMerge(query: FileMerge): Promise { + this.ensureS3OssClient(); + const { filename, md5 } = query; + const inOssFileName = await this.getInOssFileName(md5, filename); + const command = new CompleteMultipartUploadCommand({ + Bucket: this.bucket, + Key: inOssFileName, + UploadId: query.uploadId, + MultipartUpload: { Parts: query.MultipartUpload }, + }); + await this.client.send(command); + return await this.getObjectUrl(this.bucket, inOssFileName); + } } diff --git a/packages/server/src/services/file.service.ts b/packages/server/src/services/file.service.ts index 4bf1fe98..ed3ce648 100644 --- a/packages/server/src/services/file.service.ts +++ b/packages/server/src/services/file.service.ts @@ -5,7 +5,7 @@ * @Blog: https://blog.szhcloud.cn * @github: https://github.com/sang8052 * @LastEditors: SudemQaQ - * @LastEditTime: 2024-09-09 12:54:49 + * @LastEditTime: 2024-09-10 07:46:50 * @Description: */ import { Injectable } from '@nestjs/common'; @@ -18,6 +18,7 @@ import Redis from 'ioredis'; @Injectable() export class FileService { + [x: string]: any; private ossClient: OssClient; private redis: Redis; @@ -51,4 +52,16 @@ export class FileService { async mergeChunk(query) { return this.ossClient.mergeChunk(query); } + + async ossSign(query) { + return this.ossClient.ossSign(query); + } + + async ossChunk(query) { + return this.ossClient.ossChunk(query); + } + + async ossMerge(query) { + return this.ossClient.ossMerge(query); + } }