Skip to content

B
byit-myth-job
Commit 73c49b51 by huangfusuper
Options

切换fastdfs上传下载文件的方式 经fastdfs修改为集群模式

parent 784f6935
Showing with 163 additions and 90 deletions
byit-myth-admin/src/main/resources/config/application-dev.yml
......@@ -54,6 +54,16 @@ myth-job:
filestystem: FASTDFS
snapshoot-date: 60 #快照的保存时间 单位天
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
myth:
plugin:
env: ${myth-rpc.registry.env}
......
byit-myth-admin/src/main/resources/config/application-local.yml
......@@ -57,6 +57,15 @@ file:
myth-job:
filestystem: FASTDFS
snapshoot-date: 60 #快照的保存时间 单位天
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
myth:
plugin:
......
byit-myth-admin/src/main/resources/config/application-pro.yml
......@@ -51,4 +51,14 @@ file:
myth-job:
filestystem: FASTDFS
snapshoot-date: 60 #快照的保存时间 单位天
\ No newline at end of file
snapshoot-date: 60 #快照的保存时间 单位天
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
byit-myth-admin/src/main/resources/config/application-test.yml
......@@ -52,4 +52,14 @@ file:
myth-job:
filestystem: FASTDFS
snapshoot-date: 60 #快照的保存时间 单位天
\ No newline at end of file
snapshoot-date: 60 #快照的保存时间 单位天
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
byit-myth-core/myth-admin-core/src/main/java/com/byit/service/mapservice/impl/RunRecordingAndEmailServiceImpl.java
......@@ -12,7 +12,6 @@ import com.byit.service.RunRecordingService;
import com.byit.service.mapservice.RunRecordingAndEmailService;
import com.byit.util.TimeFormatUtil;
import lombok.extern.slf4j.Slf4j;
import org.csource.common.MyException;
import org.springframework.stereotype.Component;
import java.io.IOException;
......@@ -108,7 +107,7 @@ public class RunRecordingAndEmailServiceImpl implements RunRecordingAndEmailServ
if(null != jobTaskRunLog.getLogRemotelyPath()){
logStr = new String(fileSystem.downloaderFile(jobTaskRunLog.getLogRemotelyPath()), StandardCharsets.UTF_8);
}
} catch (IOException | MyException e) {
} catch (IOException e) {
e.printStackTrace();
}
stringBuilder.append("<tr align='center'>")
......
byit-myth-core/myth-core-common/pom.xml
......@@ -13,9 +13,14 @@
<artifactId>myth-core-common</artifactId>
<dependencies>
<dependency>
<!--<dependency>
<groupId>net.oschina.zcx7878</groupId>
<artifactId>fastdfs-client-java</artifactId>
</dependency>-->
<dependency>
<groupId>com.github.tobato</groupId>
<artifactId>fastdfs-client</artifactId>
<version>1.26.1-RELEASE</version>
</dependency>
<!-- slf4j -->
<dependency>
......
byit-myth-core/myth-core-common/src/main/java/com/byit/filesystem/FastDfsFileSystem.java
package com.byit.filesystem;
import cn.hutool.core.collection.CollectionUtil;
import org.csource.common.MyException;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.ClientGlobal;
import org.csource.fastdfs.StorageClient1;
import org.csource.fastdfs.TrackerClient;
import org.csource.fastdfs.TrackerServer;
import com.github.tobato.fastdfs.FdfsClientConfig;
import com.github.tobato.fastdfs.domain.MateData;
import com.github.tobato.fastdfs.domain.StorePath;
import com.github.tobato.fastdfs.proto.storage.DownloadByteArray;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
......@@ -21,74 +21,62 @@ import java.util.Set;
*/
@Component
@ConditionalOnExpression("'${myth-job.filestystem}'.equals('FASTDFS')")
@Import(FdfsClientConfig.class)
public class FastDfsFileSystem implements FileSystem {
public FastDfsFileSystem() {
try {
ClientGlobal.init("fdfs_client.conf");
} catch (IOException | MyException e) {
e.printStackTrace();
}
private final FastFileStorageClient fastFileStorageClient;
public FastDfsFileSystem(FastFileStorageClient fastFileStorageClient) {
this.fastFileStorageClient = fastFileStorageClient;
}
@Override
public String uploadFile(byte[] fileBuffer, String fileExtName, Map<String, String> mateDaTa) throws IOException, MyException {
TrackerClient trackerClient = new TrackerClient();
TrackerServer trackerServer = trackerClient.getConnection();
StorageClient1 storageClient1 = new StorageClient1(trackerServer, null);
NameValuePair[] mathList = new NameValuePair[mateDaTa.size()];
if(CollectionUtil.isNotEmpty(mateDaTa)){
Set<Map.Entry<String, String>> mateSet = mateDaTa.entrySet();
int i = 0;
for (Map.Entry<String, String> mate : mateSet) {
mathList[i] = new NameValuePair(mate.getKey(),mate.getValue());
i++;
}
}
String fileId = storageClient1.upload_file1(fileBuffer, fileExtName, mathList);
trackerServer.close();
return fileId;
public String uploadFile(byte[] fileBuffer, String fileExtName, Map<String, String> mateDaTas) {
ByteArrayInputStream byteInput = new ByteArrayInputStream(fileBuffer);
Set<MateData> mateDataSet = new HashSet<>(2);
mateDaTas.forEach((key,value) ->{
MateData mateData = new MateData();
mateData.setName(key);
mateData.setValue(value);
mateDataSet.add(mateData);
});
StorePath storePath = fastFileStorageClient.uploadFile(byteInput, fileBuffer.length, fileExtName, mateDataSet);
return storePath.getFullPath();
}
@Override
public byte[] downloaderFile(String remPath) throws IOException, MyException {
TrackerClient trackerClient = new TrackerClient();
TrackerServer trackerServer = trackerClient.getConnection();
StorageClient1 storageClient1 = new StorageClient1(trackerServer, null);
byte[] bytes = storageClient1.download_file1(remPath);
trackerServer.close();
return bytes;
public byte[] downloaderFile(String remPath) {
DownloadByteArray callback = new DownloadByteArray();
return fastFileStorageClient.downloadFile(getPathGroup(remPath), getPath(remPath), callback);
}
@Override
public void fileRemove(String remPath) throws IOException, MyException {
TrackerClient trackerClient = new TrackerClient();
TrackerServer trackerServer = trackerClient.getConnection();
StorageClient1 storageClient1 = new StorageClient1(trackerServer, null);
storageClient1.delete_file1(remPath);
trackerServer.close();
public void fileRemove(String remPath) {
fastFileStorageClient.deleteFile(remPath);
}
@Override
public Map<String, String> getFileMate(String filePath) throws IOException, MyException {
TrackerClient trackerClient = new TrackerClient();
TrackerServer trackerServer = trackerClient.getConnection();
StorageClient1 storageClient1 = new StorageClient1(trackerServer, null);
NameValuePair[] metadata1 = storageClient1.get_metadata1(filePath);
Map<String,String> map = new HashMap<String,String>(5);
if(metadata1 != null){
public Map<String, String> getFileMate(String filePath) {
Set<MateData> metadataSet = fastFileStorageClient.getMetadata(getPathGroup(filePath), getPath(filePath));
HashMap<String, String> map = new HashMap<>(2);
metadataSet.forEach(mateData -> {
map.put(mateData.getName(),mateData.getValue());
});
return map;
}
for (int i = 0; i <metadata1.length ; i++) {
map.put(metadata1[i].getName(),metadata1[i].getValue());
}
}else{
map.put("filename","filename"+filePath.substring(filePath.lastIndexOf("."),filePath.length()));
}
public static String getPathGroup(String fullPath){
int i = fullPath.indexOf("/");
return fullPath.substring(0, i);
}
return map;
public static String getPath(String fullPath){
int i = fullPath.indexOf("/");
return fullPath.substring(i+1);
}
public static void main(String[] args) {
String filePath = "sadsadsadsadsad.py";
System.out.println(filePath.substring(filePath.lastIndexOf("."), filePath.length()));
System.out.println(getPath("ddmp/M00/00/09/CgB4Al6mSVSAHi-rAAAABhCiqmE437.log"));
}
}
byit-myth-core/myth-core-common/src/main/java/com/byit/filesystem/FileSystem.java
package com.byit.filesystem;
import org.csource.common.MyException;
import java.io.IOException;
import java.util.Map;
......@@ -17,26 +16,23 @@ public interface FileSystem {
* @param mateDaTA 文件源信息
* @return 上传路径
* @throws IOException
* @throws MyException
*/
String uploadFile(byte[] fileBuffer, String fileExtName, Map<String, String> mateDaTA) throws IOException, MyException;
String uploadFile(byte[] fileBuffer, String fileExtName, Map<String, String> mateDaTA) throws IOException;
/**
* 文件下载接口
* @param remPath 远程地址
* @return 文件字节数组
* @throws IOException
* @throws MyException
*/
byte[] downloaderFile(String remPath) throws IOException, MyException;
byte[] downloaderFile(String remPath) throws IOException;
/**
* 文件删除
* @param remPath 远程地址
* @throws IOException
* @throws MyException
*/
void fileRemove(String remPath) throws IOException, MyException;
void fileRemove(String remPath) throws IOException;
/**
......@@ -44,9 +40,8 @@ public interface FileSystem {
* @param filePath
* @return
* @throws IOException
* @throws MyException
*/
Map<String,String> getFileMate(String filePath) throws IOException, MyException;
Map<String,String> getFileMate(String filePath) throws IOException;
}
byit-myth-executor/myth-executor-server/src/main/java/com/byit/service/ScriptExecutorServiceImpl.java
......@@ -17,7 +17,6 @@ import com.byit.pool.RunThreadPool;
import com.byit.rpc.remoting.provider.annotation.RpcService;
import com.byit.utils.ServiceInfoUtil;
import lombok.extern.slf4j.Slf4j;
import org.csource.common.MyException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
......@@ -145,7 +144,7 @@ public class ScriptExecutorServiceImpl implements ScriptExecutorService {
} catch (ProcessFailureException ignored){
jobRunResultDto.setReturnResult(ReturnResult.FAIL);
} catch (IOException | MyException e) {
} catch (IOException e) {
jobRunResultDto.setReturnResult(ReturnResult.FAIL);
e.printStackTrace();
}
......@@ -205,7 +204,7 @@ public class ScriptExecutorServiceImpl implements ScriptExecutorService {
//写出脚本文件
out.write(scriptByteArray);
return file.getPath();
} catch (IOException | MyException e) {
} catch (IOException e) {
e.printStackTrace();
} finally {
if(out != null){
......
byit-myth-executor/myth-executor-server/src/main/resources/application-dev.yml
......@@ -34,4 +34,14 @@ authentication:
user:
header-name: token
expire: 43200 # 外部token有效期为12小时
pub-key: client/pub.key # 解密
\ No newline at end of file
pub-key: client/pub.key # 解密
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
byit-myth-executor/myth-executor-server/src/main/resources/application-local.yml
......@@ -34,4 +34,14 @@ authentication:
user:
header-name: token
expire: 43200 # 外部token有效期为12小时
pub-key: client/pub.key # 解密
\ No newline at end of file
pub-key: client/pub.key # 解密
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
byit-myth-executor/myth-executor-server/src/main/resources/application-pro.yml
......@@ -35,4 +35,14 @@ authentication:
user:
header-name: token
expire: 43200 # 外部token有效期为12小时
pub-key: client/pub.key # 解密
\ No newline at end of file
pub-key: client/pub.key # 解密
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
byit-myth-executor/myth-executor-server/src/main/resources/application-test.yml
......@@ -35,4 +35,14 @@ authentication:
user:
header-name: token
expire: 43200 # 外部token有效期为12小时
pub-key: client/pub.key # 解密
\ No newline at end of file
pub-key: client/pub.key # 解密
fdfs:
so-timeout: 1500
connect-timeout: 600
pool:
jmx-enabled: false
tracker-list:
- 10.0.120.216:22122
- 10.0.120.217:22122
- 10.0.120.218:22122
\ No newline at end of file
demo-client/byit-demo-client/src/main/java/com/byit/job/Mains.java
package com.byit.job;
import com.byit.filesystem.FastDfsFileSystem;
import org.csource.common.MyException;
import java.io.IOException;
......@@ -12,10 +11,10 @@ import java.io.IOException;
* @date: 2019/11/20 12:18
**/
public class Mains {
public static void main(String[] args) throws InterruptedException, IOException, MyException {
FastDfsFileSystem fastDfsFileSystem = new FastDfsFileSystem();
public static void main(String[] args) {
/*FastDfsFileSystem fastDfsFileSystem = new FastDfsFileSystem(fastFileStorageClient);
byte[] bytes = fastDfsFileSystem.downloaderFile("ddmp/M00/00/00/CgB4Al5wa36ADcUtAAAAiVc2FQ85978.py");
System.out.println(new String(bytes));
System.out.println(new String(bytes));*/
/*new JobRunServerLauncher("/plugin.xml");
JobUtils.jobCache.forEach((key,value) ->{
......
pom.xml
......@@ -17,6 +17,7 @@
<module>byit-mybatis-plugin</module>
<module>byit-myth-gateway</module>
<module>byit-plugin-core</module>
<module>byit-myth-components</module>
</modules>
<parent>
......@@ -71,12 +72,20 @@
<springfox-swagger-ui.version>2.9.2</springfox-swagger-ui.version>
<org-apache-commons.version>1.3</org-apache-commons.version>
<fastdfs-client-java-version>1.27.0.0</fastdfs-client-java-version>
<fastdfs-client-version>1.26.1-RELEASE</fastdfs-client-version>
</properties>
<dependencyManagement>
<dependencies>
<!-- fastdfs-client 依赖-->
<dependency>
<groupId>com.github.tobato</groupId>
<artifactId>fastdfs-client</artifactId>
<version>${fastdfs-client-version}</version>
</dependency>
<dependency>
<groupId>net.oschina.zcx7878</groupId>
<artifactId>fastdfs-client-java</artifactId>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment