Merge 95f5f9552c into d48aa62b5e

config/dev.yaml

@@ -48,6 +48,7 @@ db:
     host: '127.0.0.1'
     port: '6379'
     password: 'root'
+    db: 0
 
 # oss 文件存储服务
 oss:
@@ -55,22 +56,19 @@ oss:
     enable: true
     # 线上更改为服务端地址（如：https://api.codingit.cn）
     server: 'http://localhost:5002'
-  # 以下为各厂商 sdk 配置，不要修改字段，填入值即可
+  s3:
-  tencent:
+    enable: true
-    enable: false
     config:
-      SecretId: ''
+        # isp 提供商,可选 minio,s3,aliyun,tencent
-      SecretKey: ''
+        cloudisp: 'minio'
-      Bucket: ''
+        accessKeyId: ''
-      Region: ''
+        secretAccessKey: ''
-  aliyun:
+        bucket: ''
-    enable: false
+        region: ''
-    config:
+        # 仅cloudisp 的值为 minio/s3 时配置，其它提供商为空
-      accessKeyId: ''
+        forcePathStyle: false
-      accessKeySecret: ''
+        # 仅cloudisp 的值为 minio 时配置，其它提供商为空
-      bucket: ''
+        endpoint: ''
-      https: true
-      region: ''
 
 # jwt 配置
 jwt:

packages/client/public/diagram/styles/default.xml

@@ -0,0 +1,203 @@
+<mxStylesheet>
+	<add as="defaultVertex">
+		<add as="shape" value="label"/>
+		<add as="perimeter" value="rectanglePerimeter"/>
+		<add as="fontSize" value="12"/>
+		<add as="fontFamily" value="Helvetica"/>
+		<add as="align" value="center"/>
+		<add as="verticalAlign" value="middle"/>
+		<add as="fillColor" value="default"/>
+		<add as="strokeColor" value="default"/>
+		<add as="fontColor" value="default"/>
+	</add>
+	<add as="defaultEdge">
+		<add as="shape" value="connector"/>
+		<add as="labelBackgroundColor" value="default"/>
+		<add as="endArrow" value="classic"/>
+		<add as="fontSize" value="11"/>
+		<add as="fontFamily" value="Helvetica"/>
+		<add as="align" value="center"/>
+		<add as="verticalAlign" value="middle"/>
+		<add as="rounded" value="1"/>
+		<add as="strokeColor" value="default"/>
+		<add as="fontColor" value="default"/>
+	</add>
+	<add as="text">
+		<add as="fillColor" value="none"/>
+		<add as="gradientColor" value="none"/>
+		<add as="strokeColor" value="none"/>
+		<add as="align" value="left"/>
+		<add as="verticalAlign" value="top"/>
+	</add>
+	<add as="edgeLabel" extend="text">
+		<add as="labelBackgroundColor" value="default"/>
+		<add as="fontSize" value="11"/>
+	</add>
+	<add as="label">
+		<add as="fontStyle" value="1"/>
+		<add as="align" value="left"/>
+		<add as="verticalAlign" value="middle"/>
+		<add as="spacing" value="2"/>
+		<add as="spacingLeft" value="52"/>
+		<add as="imageWidth" value="42"/>
+		<add as="imageHeight" value="42"/>
+		<add as="rounded" value="1"/>
+	</add>
+	<add as="icon" extend="label">
+		<add as="align" value="center"/>
+		<add as="imageAlign" value="center"/>
+		<add as="verticalLabelPosition" value="bottom"/>
+		<add as="verticalAlign" value="top"/>
+		<add as="spacingTop" value="4"/>
+		<add as="labelBackgroundColor" value="default"/>
+		<add as="spacing" value="0"/>
+		<add as="spacingLeft" value="0"/>
+		<add as="spacingTop" value="6"/>
+		<add as="fontStyle" value="0"/>
+		<add as="imageWidth" value="48"/>
+		<add as="imageHeight" value="48"/>
+	</add>
+	<add as="swimlane">
+		<add as="shape" value="swimlane"/>
+		<add as="fontSize" value="12"/>
+		<add as="fontStyle" value="1"/>
+		<add as="startSize" value="23"/>
+	</add>
+	<add as="group">
+		<add as="verticalAlign" value="top"/>
+		<add as="fillColor" value="none"/>
+		<add as="strokeColor" value="none"/>
+		<add as="gradientColor" value="none"/>
+		<add as="pointerEvents" value="0"/>
+	</add>
+	<add as="ellipse">
+		<add as="shape" value="ellipse"/>
+		<add as="perimeter" value="ellipsePerimeter"/>
+	</add>
+	<add as="rhombus">
+		<add as="shape" value="rhombus"/>
+		<add as="perimeter" value="rhombusPerimeter"/>
+	</add>
+	<add as="triangle">
+		<add as="shape" value="triangle"/>
+		<add as="perimeter" value="trianglePerimeter"/>
+	</add>
+	<add as="line">
+		<add as="shape" value="line"/>
+		<add as="strokeWidth" value="4"/>
+		<add as="labelBackgroundColor" value="default"/>
+		<add as="verticalAlign" value="top"/>
+		<add as="spacingTop" value="8"/>
+	</add>
+	<add as="image">
+		<add as="shape" value="image"/>
+		<add as="labelBackgroundColor" value="default"/>
+		<add as="verticalAlign" value="top"/>
+		<add as="verticalLabelPosition" value="bottom"/>
+	</add>
+	<add as="roundImage" extend="image">
+		<add as="perimeter" value="ellipsePerimeter"/>
+	</add>
+	<add as="rhombusImage" extend="image">
+		<add as="perimeter" value="rhombusPerimeter"/>
+	</add>
+	<add as="arrow">
+		<add as="shape" value="arrow"/>
+		<add as="edgeStyle" value="none"/>
+		<add as="fillColor" value="default"/>
+	</add>
+	<add as="fancy">
+		<add as="shadow" value="1"/>
+		<add as="glass" value="1"/>
+	</add>
+	<add as="gray" extend="fancy">
+		<add as="gradientColor" value="#B3B3B3"/>
+		<add as="fillColor" value="#F5F5F5"/>
+		<add as="strokeColor" value="#666666"/>
+	</add>
+	<add as="blue" extend="fancy">
+		<add as="gradientColor" value="#7EA6E0"/>
+		<add as="fillColor" value="#DAE8FC"/>
+		<add as="strokeColor" value="#6C8EBF"/>
+	</add>
+	<add as="green" extend="fancy">
+		<add as="gradientColor" value="#97D077"/>
+		<add as="fillColor" value="#D5E8D4"/>
+		<add as="strokeColor" value="#82B366"/>
+	</add>
+	<add as="turquoise" extend="fancy">
+		<add as="gradientColor" value="#67AB9F"/>
+		<add as="fillColor" value="#D5E8D4"/>
+		<add as="strokeColor" value="#6A9153"/>
+	</add>
+	<add as="yellow" extend="fancy">
+		<add as="gradientColor" value="#FFD966"/>
+		<add as="fillColor" value="#FFF2CC"/>
+		<add as="strokeColor" value="#D6B656"/>
+	</add>
+	<add as="orange" extend="fancy">
+		<add as="gradientColor" value="#FFA500"/>
+		<add as="fillColor" value="#FFCD28"/>
+		<add as="strokeColor" value="#D79B00"/>
+	</add>
+	<add as="red" extend="fancy">
+		<add as="gradientColor" value="#EA6B66"/>
+		<add as="fillColor" value="#F8CECC"/>
+		<add as="strokeColor" value="#B85450"/>
+	</add>
+	<add as="pink" extend="fancy">
+		<add as="gradientColor" value="#B5739D"/>
+		<add as="fillColor" value="#E6D0DE"/>
+		<add as="strokeColor" value="#996185"/>
+	</add>
+	<add as="purple" extend="fancy">
+		<add as="gradientColor" value="#8C6C9C"/>
+		<add as="fillColor" value="#E1D5E7"/>
+		<add as="strokeColor" value="#9673A6"/>
+	</add>
+	<add as="plain-gray">
+		<add as="gradientColor" value="#B3B3B3"/>
+		<add as="fillColor" value="#F5F5F5"/>
+		<add as="strokeColor" value="#666666"/>
+	</add>
+	<add as="plain-blue">
+		<add as="gradientColor" value="#7EA6E0"/>
+		<add as="fillColor" value="#DAE8FC"/>
+		<add as="strokeColor" value="#6C8EBF"/>
+	</add>
+	<add as="plain-green">
+		<add as="gradientColor" value="#97D077"/>
+		<add as="fillColor" value="#D5E8D4"/>
+		<add as="strokeColor" value="#82B366"/>
+	</add>
+	<add as="plain-turquoise">
+		<add as="gradientColor" value="#67AB9F"/>
+		<add as="fillColor" value="#D5E8D4"/>
+		<add as="strokeColor" value="#6A9153"/>
+	</add>
+	<add as="plain-yellow">
+		<add as="gradientColor" value="#FFD966"/>
+		<add as="fillColor" value="#FFF2CC"/>
+		<add as="strokeColor" value="#D6B656"/>
+	</add>
+	<add as="plain-orange">
+		<add as="gradientColor" value="#FFA500"/>
+		<add as="fillColor" value="#FFCD28"/>
+		<add as="strokeColor" value="#D79B00"/>
+	</add>
+	<add as="plain-red">
+		<add as="gradientColor" value="#EA6B66"/>
+		<add as="fillColor" value="#F8CECC"/>
+		<add as="strokeColor" value="#B85450"/>
+	</add>
+	<add as="plain-pink">
+		<add as="gradientColor" value="#B5739D"/>
+		<add as="fillColor" value="#E6D0DE"/>
+		<add as="strokeColor" value="#996185"/>
+	</add>
+	<add as="plain-purple">
+		<add as="gradientColor" value="#8C6C9C"/>
+		<add as="fillColor" value="#E1D5E7"/>
+		<add as="strokeColor" value="#9673A6"/>
+	</add>
+</mxStylesheet>

packages/domains/lib/api/file.js

@@ -35,4 +35,6 @@ exports.FileApiDefinition = {
         client: function () { return '/file/merge/chunk'; }
     }
 };
-exports.FILE_CHUNK_SIZE = 2 * 1024 * 1024;
+// 设置文件分片的大小 改成 8 M 
+// MINIO 等oss 有最小分片的限制
+exports.FILE_CHUNK_SIZE = 8 * 1024 * 1024;

packages/server/package.json

@@ -18,6 +18,8 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@aws-sdk/s3-request-presigner": "^3.645.0",
     "@hocuspocus/server": "^1.0.0-alpha.91",
     "@hocuspocus/transformer": "^1.0.0-alpha.18",
     "@nestjs/common": "^8.0.0",
@@ -61,6 +63,7 @@
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.2.0",
+    "spark-md5": "^3.0.2",
     "typeorm": "^0.2.41",
     "ua-parser-js": "^1.0.2",
     "validator": "^13.7.0",

packages/server/src/helpers/file.helper/index.ts

@@ -1,19 +1,14 @@
 import { ConfigService } from '@nestjs/config';
 
-import { AliyunOssClient } from './aliyun.client';
 import { LocalOssClient } from './local.client';
 import { OssClient } from './oss.client';
-import { TencentOssClient } from './tencent.client';
+import { S3OssClient } from './s3.client';
 
 export { OssClient };
 
 export const getOssClient = (configService: ConfigService): OssClient => {
-  if (configService.get('oss.tencent.enable')) {
+  if (configService.get('oss.s3.enable')) {
-    return new TencentOssClient(configService);
+    return new S3OssClient(configService);
-  }
-
-  if (configService.get('oss.aliyun.enable')) {
-    return new AliyunOssClient(configService);
   }
 
   return new LocalOssClient(configService);

packages/server/src/helpers/file.helper/local.client.ts

@@ -1,6 +1,7 @@
 import { FILE_CHUNK_SIZE } from '@think/domains';
 
 import * as fs from 'fs-extra';
+import Redis from 'ioredis';
 import * as path from 'path';
 
 import { BaseOssClient, FileQuery } from './oss.client';
@@ -20,6 +21,8 @@ export const pipeWriteStream = (filepath, writeStream): Promise<void> => {
 };
 
 export class LocalOssClient extends BaseOssClient {
+  private redis: Redis | null;
+
   /**
    * 文件存储路径
    * @param md5
@@ -34,6 +37,10 @@ export class LocalOssClient extends BaseOssClient {
     return { relative: filepath.replace(FILE_ROOT_PATH, FILE_DEST), absolute: filepath };
   }
 
+  async setRedis(redis: Redis) {
+    this.redis = redis;
+  }
+
   /**
    * 将文件存储的相对路径拼接为可访问 URL
    * @param serverRoot

packages/server/src/helpers/file.helper/oss.client.ts

@@ -1,5 +1,7 @@
 import { ConfigService } from '@nestjs/config';
 
+import Redis from 'ioredis';
+
 export type FileQuery = {
   filename: string;
   md5: string;
@@ -7,6 +9,7 @@ export type FileQuery = {
 };
 
 export abstract class OssClient {
+  [x: string]: any;
   abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string>;
   abstract initChunk(query: FileQuery): Promise<void | string>;
   abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void>;
@@ -39,4 +42,9 @@ export class BaseOssClient implements OssClient {
   mergeChunk(query: FileQuery): Promise<string> {
     throw new Error('Method not implemented.');
   }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  setRedis(redis: Redis): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
 }

packages/server/src/helpers/file.helper/s3.client.ts

@@ -0,0 +1,237 @@
+import {
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  PutObjectCommand,
+  S3Client,
+  UploadPartCommand,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import Redis from 'ioredis';
+
+import { BaseOssClient, FileQuery } from './oss.client';
+
+export class S3OssClient extends BaseOssClient {
+  private client: S3Client | null;
+  private bucket: string | null;
+  private redis: Redis | null;
+
+  async setRedis(redis: Redis) {
+    this.redis = redis;
+  }
+
+  /**
+   * 构建 s3 客户端
+   * @returns
+   */
+  private ensureS3OssClient(): S3Client {
+    if (this.client) {
+      return this.client;
+    }
+
+    const config = this.configService.get('oss.s3.config');
+    try {
+      this.bucket = config.bucket;
+
+      if (config.cloudisp == 'minio') {
+        this.client = new S3Client({
+          endpoint: config.endpoint,
+          region: config.region,
+          forcePathStyle: config.forcePathStyle,
+          credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+          },
+        });
+      }
+
+      if (config.cloudisp == 's3') {
+        this.client = new S3Client({
+          region: config.region,
+          forcePathStyle: config.forcePathStyle,
+          credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+          },
+        });
+      }
+
+      if (config.cloudisp == 'aliyun') {
+        this.client = new S3Client({
+          region: config.region,
+          endpoint: 'https://' + config.region + '.aliyuncs.com',
+          // 阿里云不支持 虚拟路径,这里必须为false
+          forcePathStyle: false,
+          credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+          },
+        });
+      }
+
+      if (config.cloudisp == 'tencent') {
+        this.client = new S3Client({
+          region: config.region,
+          endpoint: 'https://cos.' + config.region + '.myqcloud.com',
+          // 不支持 虚拟路径,这里必须为false
+          forcePathStyle: false,
+          credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+          },
+        });
+      }
+
+      return this.client;
+    } catch (err) {
+      console.log('无法启动S3存储服务，请检查S3配置是否正确', err.message);
+    }
+  }
+
+  /**
+   * 获取上传文件名
+   * @param md5
+   * @param filename
+   * @returns
+   */
+  private getInOssFileName(md5, filename) {
+    return `think/${md5}/${filename}`;
+  }
+
+  private async getObjectUrl(bucket, key) {
+    this.ensureS3OssClient();
+    const command = new GetObjectCommand({ Bucket: bucket, Key: key });
+    const signUrl = await getSignedUrl(this.client, command);
+    console.log('signUrl:' + signUrl);
+    return signUrl.split('?')[0];
+  }
+
+  /**
+   * 检查文件是否已存储到 oss
+   * @param md5
+   * @param filename
+   * @returns
+   */
+  private async checkIfAlreadyInOss(md5, filename) {
+    this.ensureS3OssClient();
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const command = new HeadObjectCommand({ Bucket: this.bucket, Key: inOssFileName });
+    try {
+      await this.client.send(command);
+      return await this.getObjectUrl(this.bucket, inOssFileName);
+    } catch (err) {
+      return false;
+    }
+  }
+
+  /**
+   * 上传小文件
+   * @param file
+   * @param query
+   * @returns
+   */
+  async uploadFile(file: Express.Multer.File, query: FileQuery): Promise<string> {
+    this.ensureS3OssClient();
+    const { filename, md5 } = query;
+    const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
+    if (maybeOssURL) {
+      return maybeOssURL;
+    }
+
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const command = new PutObjectCommand({
+      Bucket: this.bucket,
+      Key: inOssFileName,
+      Body: file.buffer,
+    });
+    await this.client.send(command);
+    return await this.getObjectUrl(this.bucket, inOssFileName);
+  }
+
+  /**
+   * 初始分片
+   * @param file
+   * @param query
+   * @returns
+   */
+  async initChunk(query: FileQuery): Promise<string | void> {
+    const { md5, filename } = query;
+    this.ensureS3OssClient();
+
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename);
+
+    if (maybeOssURL) {
+      return maybeOssURL as string;
+    }
+    const command = new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: inOssFileName });
+    const response = await this.client.send(command);
+    const upload_id = response['UploadId'];
+    // 这里使用redis 来存储 upload_id
+    await this.redis.del('think:oss:chunk:' + md5);
+    await this.redis.del('think:oss:chunk:' + md5 + '*');
+    this.redis.set('think:oss:chunk:' + md5, upload_id);
+    return '';
+  }
+
+  /**
+   * 上传分片
+   * @param file
+   * @param query
+   * @returns
+   */
+  async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise<void> {
+    const { md5, filename, chunkIndex } = query;
+
+    if (!('chunkIndex' in query)) {
+      throw new Error('请指定 chunkIndex');
+    }
+
+    this.ensureS3OssClient();
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const upload_id = await this.redis.get('think:oss:chunk:' + md5);
+
+    const command = new UploadPartCommand({
+      Body: file.buffer,
+      Bucket: this.bucket,
+      Key: inOssFileName,
+      PartNumber: chunkIndex,
+      UploadId: upload_id,
+    });
+    const response = await this.client.send(command);
+    this.redis.set(
+      'think:oss:chunk:' + md5 + ':' + chunkIndex,
+      JSON.stringify({ PartNumber: chunkIndex, ETag: response['ETag'] })
+    );
+  }
+
+  /**
+   * 合并分片
+   * @param query
+   * @returns
+   */
+  async mergeChunk(query: FileQuery): Promise<string> {
+    const { filename, md5 } = query;
+    const inOssFileName = this.getInOssFileName(md5, filename);
+    const upload_id = await this.redis.get('think:oss:chunk:' + md5);
+    const etags = await this.redis.keys('think:oss:chunk:' + md5 + ':*');
+    const MultipartUpload = { Parts: [] };
+    for (let i = 1; i <= etags.length; i++) {
+      const obj = JSON.parse(await this.redis.get('think:oss:chunk:' + md5 + ':' + i));
+      MultipartUpload.Parts.push(obj);
+    }
+    console.log(MultipartUpload, upload_id);
+    const command = new CompleteMultipartUploadCommand({
+      Bucket: this.bucket,
+      Key: inOssFileName,
+      UploadId: upload_id,
+      MultipartUpload: MultipartUpload,
+    });
+
+    await this.client.send(command);
+    await this.redis.del('think:oss:chunk:' + md5);
+    await this.redis.del('think:oss:chunk:' + md5 + '*');
+    return await this.getObjectUrl(this.bucket, inOssFileName);
+  }
+}

packages/server/src/helpers/redis.helper.ts

@@ -18,7 +18,7 @@ export const buildRedis = (db: RedisDBEnum): Promise<Redis> => {
       ...redisConfig,
       showFriendlyErrorStack: true,
       lazyConnect: true,
-      db,
+      db: redisConfig.db,
     });
     redis.on('ready', () => {
       resolve(redis);

packages/server/src/services/file.service.ts

@@ -1,14 +1,39 @@
+/*
+ * @Author: SudemQaQ
+ * @Date: 2024-09-09 10:28:02
+ * @email: mail@szhcloud.cn
+ * @Blog: https://blog.szhcloud.cn
+ * @github: https://github.com/sang8052
+ * @LastEditors: SudemQaQ
+ * @LastEditTime: 2024-09-09 12:54:49
+ * @Description:
+ */
 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 
+import { RedisDBEnum } from '@constants/*';
 import { getOssClient, OssClient } from '@helpers/file.helper';
+import { buildRedis } from '@helpers/redis.helper';
+import Redis from 'ioredis';
 
 @Injectable()
 export class FileService {
   private ossClient: OssClient;
+  private redis: Redis;
 
   constructor(private readonly configService: ConfigService) {
     this.ossClient = getOssClient(this.configService);
+    this.buildRedis();
+  }
+
+  private async buildRedis() {
+    try {
+      this.redis = await buildRedis(RedisDBEnum.view);
+      console.log('[think] 文件服务启动成功');
+      this.ossClient.setRedis(this.redis);
+    } catch (e) {
+      console.error(`[think] 文件服务启动错误: "${e.message}"`);
+    }
   }
 
   async uploadFile(file, query) {