Compare commits

...

2 Commits

Author SHA1 Message Date
桑泽寰 79414e9319
Merge 8500370121 into d48aa62b5e 2024-09-10 14:18:57 +08:00
sudemqaq 8500370121 1.目前支持前端通过s3 协议直传到对象存储
2.微调了前端上传图片界面时显示的图片被错误的设置成宽高比均为100% 的问题,修正为100 % 的宽度
3.调整前端下载文件的请求改为url 地址以 http:// 或 https://开头时自动重定向下载,否则使用 FileSaver 下载文件
2024-09-10 13:18:56 +08:00
13 changed files with 365 additions and 75 deletions

View File

@ -73,4 +73,4 @@ oss:
# jwt 配置 # jwt 配置
jwt: jwt:
secretkey: 'zA_Think+KNOWLEDGE+WIKI+DOCUMENTS@2022' secretkey: 'zA_Think+KNOWLEDGE+WIKI+DOCUMENTS@2022'
expiresIn: '6h' expiresIn: '6h'

View File

@ -23,7 +23,7 @@ const nextConfig = semi({
env: { env: {
SERVER_API_URL: config.client.apiUrl, SERVER_API_URL: config.client.apiUrl,
COLLABORATION_API_URL: config.client.collaborationUrl, COLLABORATION_API_URL: config.client.collaborationUrl,
ENABLE_ALIYUN_OSS: !!config.oss.aliyun.accessKeyId, ENABLE_OSS_S3: config.oss.s3.enable,
DNS_PREFETCH: (config.client.dnsPrefetch || '').split(' '), DNS_PREFETCH: (config.client.dnsPrefetch || '').split(' '),
SEO_APPNAME: config.client.seoAppName, SEO_APPNAME: config.client.seoAppName,
SEO_DESCRIPTION: config.client.seoDescription, SEO_DESCRIPTION: config.client.seoDescription,

View File

@ -9,8 +9,8 @@ import styles from './style.module.scss';
type ISize = { width: number; height: number }; type ISize = { width: number; height: number };
interface IProps { interface IProps {
width: number; width: number | string;
height: number; height: number | string;
maxWidth?: number; maxWidth?: number;
isEditable?: boolean; isEditable?: boolean;
onChange?: (arg: ISize) => void; onChange?: (arg: ISize) => void;

View File

@ -1,6 +1,11 @@
import { Toast } from '@douyinfe/semi-ui';
import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains'; import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains';
import axios from 'axios';
import { url } from 'inspector';
import { string } from 'lib0'; import { string } from 'lib0';
import { timeout } from 'lib0/eventloop';
import SparkMD5 from 'spark-md5'; import SparkMD5 from 'spark-md5';
import { HttpClient } from './http-client'; import { HttpClient } from './http-client';
@ -35,7 +40,6 @@ const uploadFileToServer = (arg: {
}) => { }) => {
const { filename, file, md5, isChunk, chunkIndex, onUploadProgress } = arg; const { filename, file, md5, isChunk, chunkIndex, onUploadProgress } = arg;
const api = isChunk ? 'uploadChunk' : 'upload'; const api = isChunk ? 'uploadChunk' : 'upload';
const formData = new FormData(); const formData = new FormData();
formData.append('file', file); formData.append('file', file);
@ -51,6 +55,7 @@ const uploadFileToServer = (arg: {
md5, md5,
chunkIndex, chunkIndex,
}, },
timeout: 30 * 1000,
onUploadProgress: (progress) => { onUploadProgress: (progress) => {
const percent = progress.loaded / progress.total; const percent = progress.loaded / progress.total;
onUploadProgress && onUploadProgress(percent); onUploadProgress && onUploadProgress(percent);
@ -67,68 +72,152 @@ export const uploadFile = async (
return onUploadProgress && onUploadProgress(Math.ceil(percent * 100)); return onUploadProgress && onUploadProgress(Math.ceil(percent * 100));
}; };
const filename = file.name; // 开启s3 文件上传支持
if (!process.env.ENABLE_OSS_S3) {
const filename = file.name;
console.debug('当前没有开启oss 对象存储,使用本地上传方案');
if (file.size > FILE_CHUNK_SIZE) {
onTooLarge && onTooLarge();
}
if (file.size > FILE_CHUNK_SIZE * 5) { if (file.size <= FILE_CHUNK_SIZE) {
onTooLarge && onTooLarge(); const spark = new SparkMD5();
} spark.append(file);
spark.append(file.lastModified);
spark.append(file.type);
const md5 = spark.end();
const url = await uploadFileToServer({ filename, file, md5, onUploadProgress: wraponUploadProgress });
return url;
} else {
const { chunks, md5 } = await splitBigFile(file);
const unitPercent = 1 / chunks.length;
const progressMap = {};
if (file.size <= FILE_CHUNK_SIZE) { let url = await HttpClient.request<string | undefined>({
const spark = new SparkMD5(); method: FileApiDefinition.initChunk.method,
spark.append(file); url: FileApiDefinition.initChunk.client(),
spark.append(file.lastModified);
spark.append(file.type);
const md5 = spark.end();
const url = await uploadFileToServer({ filename, file, md5, onUploadProgress: wraponUploadProgress });
return url;
} else {
const { chunks, md5 } = await splitBigFile(file);
const unitPercent = 1 / chunks.length;
const progressMap = {};
let url = await HttpClient.request<string | undefined>({
method: FileApiDefinition.initChunk.method,
url: FileApiDefinition.initChunk.client(),
params: {
filename,
md5,
},
});
if (!url) {
await Promise.all(
chunks.map((chunk, index) => {
return uploadFileToServer({
filename,
file: chunk,
chunkIndex: index + 1,
md5,
isChunk: true,
onUploadProgress: (progress) => {
progressMap[index] = progress * unitPercent;
wraponUploadProgress(
Math.min(
Object.keys(progressMap).reduce((a, c) => {
return (a += progressMap[c]);
}, 0),
// 剩下的 5% 交给 merge
0.95
)
);
},
});
})
);
url = await HttpClient.request({
method: FileApiDefinition.mergeChunk.method,
url: FileApiDefinition.mergeChunk.client(),
params: { params: {
filename, filename,
md5, md5,
}, },
}); });
if (!url) {
await Promise.all(
chunks.map((chunk, index) => {
return uploadFileToServer({
filename,
file: chunk,
chunkIndex: index + 1,
md5,
isChunk: true,
onUploadProgress: (progress) => {
progressMap[index] = progress * unitPercent;
wraponUploadProgress(
Math.min(
Object.keys(progressMap).reduce((a, c) => {
return (a += progressMap[c]);
}, 0),
// 剩下的 5% 交给 merge
0.95
)
);
},
});
})
);
url = await HttpClient.request({
method: FileApiDefinition.mergeChunk.method,
url: FileApiDefinition.mergeChunk.client(),
params: {
filename,
md5,
},
});
}
wraponUploadProgress(1);
return url;
}
}
// S3 后端签名 前端文件直传 方案
else {
// 前端计算文件的md5
console.log('计算待上传的文件{' + file.name + '}的md5...');
const { chunks, md5 } = await splitBigFile(file);
console.log('文件{' + file.name + '}的md5:' + md5);
const filename = file.name;
// 请求后端检查指定的文件是不是已经存在
const res = await HttpClient.request({
method: FileApiDefinition.ossSign.method,
url: FileApiDefinition.ossSign.client(),
data: { filename, md5, fileSize: file.size },
});
// 如果后端反应文件已经存在
if (res['isExist']) {
Toast.info('文件秒传成功!');
return res['objectUrl'];
} else {
//console.log('文件不存在,需要上传文件');
// 后端认为文件小,前端直接put 上传
if (!res['MultipartUpload']) {
console.log('前端直接PUT上传文件');
const signUrl = res['signUrl'];
await axios.put(signUrl, file, {
timeout: 120 * 1000,
onUploadProgress: (process) => {
const uploadLoaded = process.loaded;
const uploadTotal = file.size;
const uploadPercent = uploadLoaded / uploadTotal;
wraponUploadProgress(uploadPercent);
},
});
const upres = await HttpClient.request({
method: FileApiDefinition.ossSign.method,
url: FileApiDefinition.ossSign.client(),
data: { filename, md5, fileSize: file.size },
});
return upres['objectUrl'];
}
// 前端进入分片上传流程
else {
const upload_id = res['uploadId'];
// console.log('分片文件上传,upload_id:' + upload_id);
const MultipartUpload = [];
for (let index = 0; index < chunks.length; index++) {
const chunk = chunks[index];
const res = await HttpClient.request({
method: FileApiDefinition.ossChunk.method,
url: FileApiDefinition.ossChunk.client(),
data: { filename, md5, uploadId: upload_id, chunkIndex: index + 1 },
});
// 上传文件分块到s3
// 直接用原生请求不走拦截器
const upload_res = await axios.put(res['signUrl'], chunk, {
timeout: 120 * 1000,
onUploadProgress: (process) => {
const uploadLoaded = process.loaded + FILE_CHUNK_SIZE * index;
const uploadTotal = file.size;
const uploadPercent = uploadLoaded / uploadTotal;
//console.log(uploadLoaded, uploadTotal, uploadPercent);
wraponUploadProgress(uploadPercent);
},
});
const upload_etag = upload_res.headers['etag'];
const response_part = { PartNumber: index + 1, ETag: upload_etag };
MultipartUpload.push(response_part);
//console.log('文件分片{' + (index + 1) + '上传成功etag:' + upload_etag);
}
// 文件已经全部上传OK
// 请求后端合并文件
const payload = { filename, md5, uploadId: upload_id, MultipartUpload };
const upres = await HttpClient.request({
method: FileApiDefinition.ossMerge.method,
url: FileApiDefinition.ossMerge.client(),
data: payload,
});
return '' + upres;
}
} }
wraponUploadProgress(1);
return url;
} }
}; };

View File

@ -39,6 +39,7 @@ HttpClient.interceptors.response.use(
isBrowser && Toast.error(data.data.message); isBrowser && Toast.error(data.data.message);
return null; return null;
} }
// 如果是 204 请求 那么直接返回 data.headers
const res = data.data; const res = data.data;

View File

@ -88,9 +88,8 @@ export const ImageWrapper = ({ editor, node, updateAttributes }) => {
<NodeViewWrapper style={{ textAlign, fontSize: 0, maxWidth: '100%' }}> <NodeViewWrapper style={{ textAlign, fontSize: 0, maxWidth: '100%' }}>
<Resizeable <Resizeable
className={'render-wrapper'} className={'render-wrapper'}
width={width || maxWidth} width="100%"
height={height} height="100%"
maxWidth={maxWidth}
isEditable={isEditable} isEditable={isEditable}
onChangeEnd={onResize} onChangeEnd={onResize}
> >
@ -106,11 +105,9 @@ export const ImageWrapper = ({ editor, node, updateAttributes }) => {
</Spin> </Spin>
</div> </div>
) : ( ) : (
<div className={styles.wrap}> <div className={styles.wrap} title="parent_image_div">
<div <div
style={{ style={{
height: '100%',
maxHeight: '100%',
padding: 24, padding: 24,
overflow: 'hidden', overflow: 'hidden',
display: 'flex', display: 'flex',
@ -120,7 +117,7 @@ export const ImageWrapper = ({ editor, node, updateAttributes }) => {
transition: `all ease-in-out .3s`, transition: `all ease-in-out .3s`,
}} }}
> >
<LazyLoadImage src={src} alt={alt} width={'100%'} height={'100%'} /> <LazyLoadImage src={src} alt={alt} width={'100%'} />
</div> </div>
<div className={styles.handlerWrap}> <div className={styles.handlerWrap}>

View File

@ -1,5 +1,6 @@
import FileSaver from 'file-saver'; import FileSaver from 'file-saver';
export function download(url, name) { export function download(url, name) {
FileSaver.saveAs(url, name); if (url.startsWith('http://') || url.startsWith('https://')) window.open(url, '文件下载...');
else FileSaver.saveAs(url, name);
} }

View File

@ -1,3 +1,4 @@
export declare const FileApiDefinition: { export declare const FileApiDefinition: {
/** /**
* *
@ -31,5 +32,33 @@ export declare const FileApiDefinition: {
server: "merge/chunk"; server: "merge/chunk";
client: () => string; client: () => string;
}; };
/**
*
*/
ossSign:{
method: "post";
server: "upload/ossSign";
client: () => string;
};
/**
*
*/
ossChunk:{
method: "post";
server: "upload/ossChunk";
client: () => string;
};
/**
*
*/
ossMerge:{
method: "post";
server: "upload/ossMerge";
client: () => string;
};
}; };
export declare const FILE_CHUNK_SIZE: number; export declare const FILE_CHUNK_SIZE: number;

View File

@ -33,7 +33,22 @@ exports.FileApiDefinition = {
method: 'post', method: 'post',
server: 'merge/chunk', server: 'merge/chunk',
client: function () { return '/file/merge/chunk'; } client: function () { return '/file/merge/chunk'; }
} },
ossSign:{
method: "post",
server: "upload/ossSign",
client: function () { return '/file/upload/ossSign'; }
},
ossChunk:{
method: "post",
server: "upload/ossChunk",
client: function () { return '/file/upload/ossChunk'; }
},
ossMerge:{
method: "post",
server: "upload/ossMerge",
client: function () { return '/file/upload/ossMerge'; }
},
}; };
// 设置文件分片的大小 改成 8 M // 设置文件分片的大小 改成 8 M
// MINIO 等oss 有最小分片的限制 // MINIO 等oss 有最小分片的限制

View File

@ -1,10 +1,10 @@
import { Controller, Post, Query, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common'; import { Body, Controller, Post, Query, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express'; import { FileInterceptor } from '@nestjs/platform-express';
import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains'; import { FILE_CHUNK_SIZE, FileApiDefinition } from '@think/domains';
import { JwtGuard } from '@guard/jwt.guard'; import { JwtGuard } from '@guard/jwt.guard';
import { FileQuery } from '@helpers/file.helper/oss.client'; import { FileMerge, FileQuery } from '@helpers/file.helper/oss.client';
import { FileService } from '@services/file.service'; import { FileService } from '@services/file.service';
@Controller('file') @Controller('file')
@ -64,4 +64,31 @@ export class FileController {
mergeChunk(@Query() query: FileQuery) { mergeChunk(@Query() query: FileQuery) {
return this.fileService.mergeChunk(query); return this.fileService.mergeChunk(query);
} }
/**
*
*/
@Post(FileApiDefinition.ossSign.server)
@UseGuards(JwtGuard)
ossSign(@Body() data: FileQuery) {
return this.fileService.ossSign(data);
}
/**
*
*/
@Post(FileApiDefinition.ossChunk.server)
@UseGuards(JwtGuard)
ossChunk(@Body() data: FileQuery) {
return this.fileService.ossChunk(data);
}
/**
*
*/
@Post(FileApiDefinition.ossMerge.server)
@UseGuards(JwtGuard)
ossMerge(@Body() data: FileMerge) {
return this.fileService.ossMerge(data);
}
} }

View File

@ -1,11 +1,42 @@
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import exp from 'constants';
import Redis from 'ioredis'; import Redis from 'ioredis';
export type FileQuery = { export type FileQuery = {
filename: string; filename: string;
md5: string; md5: string;
chunkIndex?: number; chunkIndex?: number;
fileSize?: number;
uploadId?: string;
};
export type FileMerge = {
filename: string;
md5: string;
uploadId: string;
MultipartUpload: any;
};
export type chunkUpload = {
uploadId: string;
chunkIndex: number;
etag: string;
};
export type ossSignReponse = {
MultipartUpload: boolean;
isExist: boolean;
uploadId: string | null;
objectKey: string;
objectUrl: string | null;
signUrl: string | null;
};
export type ossChunkResponse = {
signUrl: string;
uploadId: string;
chunkIndex: number;
}; };
export abstract class OssClient { export abstract class OssClient {
@ -14,6 +45,9 @@ export abstract class OssClient {
abstract initChunk(query: FileQuery): Promise<void | string>; abstract initChunk(query: FileQuery): Promise<void | string>;
abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>; abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
abstract mergeChunk(query: FileQuery): Promise<string>; abstract mergeChunk(query: FileQuery): Promise<string>;
abstract ossSign(query: FileQuery): Promise<ossSignReponse>;
abstract ossChunk(query: FileQuery): Promise<ossChunkResponse>;
abstract ossMerge(query: FileMerge): Promise<string>;
} }
export class BaseOssClient implements OssClient { export class BaseOssClient implements OssClient {
@ -43,6 +77,21 @@ export class BaseOssClient implements OssClient {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');
} }
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ossSign(query: FileQuery): Promise<ossSignReponse> {
throw new Error('Method not implemented.');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ossMerge(query: FileMerge): Promise<string> {
throw new Error('Method not implemented.');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ossChunk(query: FileQuery): Promise<ossChunkResponse> {
throw new Error('Method not implemented.');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
setRedis(redis: Redis): Promise<void> { setRedis(redis: Redis): Promise<void> {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');

View File

@ -1,3 +1,5 @@
import { FILE_CHUNK_SIZE } from '@think/domains';
import { import {
CompleteMultipartUploadCommand, CompleteMultipartUploadCommand,
CreateMultipartUploadCommand, CreateMultipartUploadCommand,
@ -10,7 +12,7 @@ import {
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import Redis from 'ioredis'; import Redis from 'ioredis';
import { BaseOssClient, FileQuery } from './oss.client'; import { BaseOssClient, FileMerge, FileQuery, ossChunkResponse, ossSignReponse } from './oss.client';
export class S3OssClient extends BaseOssClient { export class S3OssClient extends BaseOssClient {
private client: S3Client | null; private client: S3Client | null;
@ -103,7 +105,6 @@ export class S3OssClient extends BaseOssClient {
this.ensureS3OssClient(); this.ensureS3OssClient();
const command = new GetObjectCommand({ Bucket: bucket, Key: key }); const command = new GetObjectCommand({ Bucket: bucket, Key: key });
const signUrl = await getSignedUrl(this.client, command); const signUrl = await getSignedUrl(this.client, command);
console.log('signUrl:' + signUrl);
return signUrl.split('?')[0]; return signUrl.split('?')[0];
} }
@ -221,7 +222,6 @@ export class S3OssClient extends BaseOssClient {
const obj = JSON.parse(await this.redis.get('think:oss:chunk:' + md5 + ':' + i)); const obj = JSON.parse(await this.redis.get('think:oss:chunk:' + md5 + ':' + i));
MultipartUpload.Parts.push(obj); MultipartUpload.Parts.push(obj);
} }
console.log(MultipartUpload, upload_id);
const command = new CompleteMultipartUploadCommand({ const command = new CompleteMultipartUploadCommand({
Bucket: this.bucket, Bucket: this.bucket,
Key: inOssFileName, Key: inOssFileName,
@ -234,4 +234,73 @@ export class S3OssClient extends BaseOssClient {
await this.redis.del('think:oss:chunk:' + md5 + '*'); await this.redis.del('think:oss:chunk:' + md5 + '*');
return await this.getObjectUrl(this.bucket, inOssFileName); return await this.getObjectUrl(this.bucket, inOssFileName);
} }
async ossSign(query: FileQuery): Promise<ossSignReponse> {
const { filename, md5, fileSize } = query;
const inOssFileName = await this.getInOssFileName(md5, filename);
this.ensureS3OssClient();
const objectUrl = await this.checkIfAlreadyInOss(md5, filename);
if (objectUrl) {
return {
signUrl: null,
MultipartUpload: false,
uploadId: null,
objectKey: inOssFileName,
isExist: true,
objectUrl: objectUrl,
};
}
if (fileSize <= FILE_CHUNK_SIZE) {
const command = new PutObjectCommand({ Bucket: this.bucket, Key: inOssFileName });
const signUrl = await getSignedUrl(this.client, command);
return {
signUrl: signUrl,
MultipartUpload: false,
uploadId: null,
objectKey: inOssFileName,
isExist: false,
objectUrl: null,
};
} else {
const command = new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: inOssFileName });
const response = await this.client.send(command);
const upload_id = response['UploadId'];
return {
signUrl: null,
MultipartUpload: true,
uploadId: upload_id,
objectKey: inOssFileName,
isExist: false,
objectUrl: null,
};
}
}
async ossChunk(query: FileQuery): Promise<ossChunkResponse> {
this.ensureS3OssClient();
const { filename, md5 } = query;
const inOssFileName = await this.getInOssFileName(md5, filename);
const command = new UploadPartCommand({
UploadId: query.uploadId,
Bucket: this.bucket,
Key: inOssFileName,
PartNumber: query.chunkIndex,
});
const signUrl = await getSignedUrl(this.client, command);
return { signUrl: signUrl, uploadId: query.uploadId, chunkIndex: query.chunkIndex };
}
async ossMerge(query: FileMerge): Promise<string> {
this.ensureS3OssClient();
const { filename, md5 } = query;
const inOssFileName = await this.getInOssFileName(md5, filename);
const command = new CompleteMultipartUploadCommand({
Bucket: this.bucket,
Key: inOssFileName,
UploadId: query.uploadId,
MultipartUpload: { Parts: query.MultipartUpload },
});
await this.client.send(command);
return await this.getObjectUrl(this.bucket, inOssFileName);
}
} }

View File

@ -5,7 +5,7 @@
* @Blog: https://blog.szhcloud.cn * @Blog: https://blog.szhcloud.cn
* @github: https://github.com/sang8052 * @github: https://github.com/sang8052
* @LastEditors: SudemQaQ * @LastEditors: SudemQaQ
* @LastEditTime: 2024-09-09 12:54:49 * @LastEditTime: 2024-09-10 07:46:50
* @Description: * @Description:
*/ */
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
@ -18,6 +18,7 @@ import Redis from 'ioredis';
@Injectable() @Injectable()
export class FileService { export class FileService {
[x: string]: any;
private ossClient: OssClient; private ossClient: OssClient;
private redis: Redis; private redis: Redis;
@ -51,4 +52,16 @@ export class FileService {
async mergeChunk(query) { async mergeChunk(query) {
return this.ossClient.mergeChunk(query); return this.ossClient.mergeChunk(query);
} }
async ossSign(query) {
return this.ossClient.ossSign(query);
}
async ossChunk(query) {
return this.ossClient.ossChunk(query);
}
async ossMerge(query) {
return this.ossClient.ossMerge(query);
}
} }