Merge remote-tracking branch 'origin/developer' into developer

byit-myth-admin/src/main/resources/application-dev.yml

 spring:
   datasource:
     driver-class-name: com.mysql.jdbc.Driver
-    url: jdbc:mysql://10.0.10.118:3306/myth-job?Unicode=true&characterEncoding=UTF-8&useSSL=true
+    url: jdbc:mysql://10.0.120.30:3307/myth-job?Unicode=true&characterEncoding=UTF-8&useSSL=true
     username: root
-    password: 123456
+    password: root
 
   mail:
     host: smtp.163.com

byit-myth-admin/src/main/resources/application-local.yml

+spring:
+  datasource:
+    driver-class-name: com.mysql.jdbc.Driver
+    url: jdbc:mysql://10.0.10.118:3306/myth-job?Unicode=true&characterEncoding=UTF-8&useSSL=true
+    username: root
+    password: 123456
+
+  mail:
+    host: smtp.163.com
+    username: huangfusuper@163.com
+    password: huangfu0110
+    default-encoding: UTF-8
+    protocol: smtp
+    properties:
+      from: huangfusuper@163.com
+
+mybatis:
+  mapper-locations: /mapper/*.xml
+
+myth-rpc:
+  registry:
+    address: http://localhost:8080/myth-register
+    env: huangfu
+    biz: byit-myth-job
+logging:
+  path: /data/mythjob
+  file: myth_log_file
+
+file:
+  system:
+    ip: 10.0.120.2
+    port: 88
+
+myth-job:
+  filestystem: FASTDFS
\ No newline at end of file

byit-myth-admin/src/main/resources/application.yml

@@ -3,7 +3,7 @@ server:
   context-path: /myth-job-admin
 spring:
   profiles:
-    active: dev
+    active: local
   application:
     name: myth-job-admin
 

byit-myth-core/myth-admin-core/src/main/java/com/byit/factory/DaemonScanThreadRunHelper.java

@@ -123,7 +123,7 @@ public class DaemonScanThreadRunHelper {
                         conn = dataSource.getConnection();
                         connAutoCommit = conn.getAutoCommit();
                         conn.setAutoCommit(false);
-                        preparedStatement = conn.prepareStatement("SELECT * FROM JOB_LOCK WHERE LOCK_NAME = '"+lockName+"' FOR UPDATE ");
+                        preparedStatement = conn.prepareStatement("SELECT * FROM job_lock WHERE LOCK_NAME = '"+lockName+"' FOR UPDATE ");
                         preparedStatement.execute();
                         //调用业务操作
                         sleepTime = examplesValue.start();

byit-myth-core/myth-admin-core/src/main/java/com/byit/thread/helper/EndAndNotWarningThreadRunHelper.java

@@ -61,14 +61,14 @@ public class EndAndNotWarningThreadRunHelper extends BaseThreadRunHelper {
                 switch (runRecording.getAlarmlAction()) {
                     //设置为完成时告警
                     case WHEN_DONE:
-                        log.info("------工作流{},被设置为完成时告警-----",runRecording);
+                        log.debug("------工作流{},被设置为完成时告警-----",runRecording);
                         if (RunRecordingEnum.FLOW_STATUS_IS_END.getCode().equals(runRecording.getFlowStatus())) {
                             runRecordingAndEmailService.saveEmailAndRunRecording(runRecording);
                         }
                         break;
                     //失败时告警
                     case FAILURE_DONE:
-                        log.info("------工作流{},被设置为失败时告警-----",runRecording);
+                        log.debug("------工作流{},被设置为失败时告警-----",runRecording);
                         if (RunRecordingEnum.RUN_FLOW_FAILURE.getCode().equals(runRecording.getFlowRunResult())
                                 || RunRecordingEnum.RUN_FLOW_RE_FAILURE.getCode().equals(runRecording.getFlowRunResult())) {
                             runRecordingAndEmailService.saveEmailAndRunRecording(runRecording);
@@ -76,14 +76,14 @@ public class EndAndNotWarningThreadRunHelper extends BaseThreadRunHelper {
                         break;
                     //成功时告警
                     case SUCCESS_DONE:
-                        log.info("------工作流{},被设置为成功时告警-----",runRecording);
+                        log.debug("------工作流{},被设置为成功时告警-----",runRecording);
                         if (RunRecordingEnum.RUN_FLOW_SUCCESS.getCode().equals(runRecording.getFlowRunResult())
                                 || RunRecordingEnum.RUN_FLOW_RE_SUCCESS.getCode().equals(runRecording.getFlowRunResult())) {
                             runRecordingAndEmailService.saveEmailAndRunRecording(runRecording);
                         }
                         break;
                     default:
-                        log.info("------工作流{},告警类别未知-----",runRecording);
+                        log.debug("------工作流{},告警类别未知-----",runRecording);
                         break;
                 }
             });

byit-myth-core/myth-admin-core/src/main/java/com/byit/thread/helper/ScheduleThreadRunHelper.java

@@ -46,32 +46,32 @@ public class ScheduleThreadRunHelper extends BaseThreadRunHelper {
         //查询所有符合条件的任务节点
         List<JobTaskSchedule> jobTaskSchedules = jobTaskScheduleService.findJobTaskScheduleByTriggerNextTimeLessThanEqual(nowTime + SCHEDULE_READ_MS);
         if(CollectionUtil.isNotEmpty(jobTaskSchedules)){
-            log.info("------排期表查询到有需要存在的节点--------");
+            log.debug("------排期表查询到有需要存在的节点--------");
             //循环遍历添加任务
             jobTaskSchedules.forEach(mythJobTaskSchedule ->{
                 mythJobTaskSchedule.setRunParam(PlaceholderUtils.formatParam(mythJobTaskSchedule.getRunParam()));
                 //如果是重跑就有logId
                 Integer logId = mythJobTaskSchedule.getLogId();
                 if(logId == null){
-                    log.info("-------------发现节点{}不是重跑,执行日志节点数据初始化-----------",mythJobTaskSchedule);
+                    log.debug("-------------发现节点{}不是重跑,执行日志节点数据初始化-----------",mythJobTaskSchedule);
                     logId = saveLog(mythJobTaskSchedule);
-                    log.info("------------{}节点的日志保存成功,日志ID为{}---------------",mythJobTaskSchedule,logId);
+                    log.debug("------------{}节点的日志保存成功,日志ID为{}---------------",mythJobTaskSchedule,logId);
                 }
                 mythJobTaskSchedule.setLogId(logId);
                 Long triggerTime = mythJobTaskSchedule.getTriggerTime();
                 TimerTask timerTask = null;
                 if (NodeTypeEnum.JAVA.getType().equals(mythJobTaskSchedule.getJobType())) {
-                    log.info("------节点{}，开始构建java执行器-------",mythJobTaskSchedule);
+                    log.debug("------节点{}，开始构建java执行器-------",mythJobTaskSchedule);
                     //构建调度执行器
                     timerTask = new JavaBeanJobTask(mythJobTaskSchedule);
                 }else if(NodeTypeEnum.SCRIPT.getType().equals(mythJobTaskSchedule.getJobType())){
-                    log.info("------节点{}，开始构建脚本执行器-------",mythJobTaskSchedule);
+                    log.debug("------节点{}，开始构建脚本执行器-------",mythJobTaskSchedule);
                     //构建脚本调度执行器
                     timerTask = new ScriptExecutorJobTask(mythJobTaskSchedule);
                 }
                 //TODO 有个坑 如果这个类型不存在的话  这个节点就不会被执行和删除  有没有办法能够强制必须有类型呢？
                 if(timerTask != null){
-                    log.info("------节点{}的执行器{}执行添加到任务调度轮的操作-------",mythJobTaskSchedule,timerTask);
+                    log.debug("------节点{}的执行器{}执行添加到任务调度轮的操作-------",mythJobTaskSchedule,timerTask);
                     WorkRoulette.addJob(timerTask,triggerTime);
                     jobTaskScheduleService.delete(mythJobTaskSchedule.getId());
                 }

byit-myth-core/myth-admin-core/src/main/java/com/byit/thread/helper/TaskThreadRunHelper.java

@@ -158,7 +158,7 @@ public class TaskThreadRunHelper extends BaseThreadRunHelper {
                     log.debug("--------------【{}的上级节点的失败节点已经全部重试完毕】------------------",thisJobTask);
                     //该节点如果为弱引用
                     if (NodePropertyEnum.WEAK_NODE.getCode().equals(thisJobTask.getSuperSuccessRun())) {
-                        log.info("-------------【查询到有弱引用节点】-----------------");
+                        log.debug("-------------【查询到有弱引用节点】-----------------");
                         //执行代码
                         runJobTask(thisJobTask,jobTaskSchedules);
                     }else{

byit-myth-executor/myth-executor-server/src/main/resources/application-local.yml

+# myth-rpc
+myth-rpc:
+  registry:
+    address: http://localhost:8080/myth-register
+    biz: byit-myth-job
+    env: huangfu
+  remoting:
+    port: 7776
+logging:
+  config: classpath:logback.xml
+myth-job:
+  filestystem: FASTDFS
+  log:
+    root:
+      path: /mythjob/log/
+  script:
+    root:
+      path: /mythjob/script/
+
+spring:
+  redis:
+    database: 0
+    host: 10.0.120.30
+    port: 6379
+    password:
+    pool:
+      max-active: 8
+      max-wait: 1
+      max-idle: 8
+      min-idle: 0
+    timeout: 0

byit-myth-executor/myth-executor-server/src/main/resources/application.yml

@@ -5,4 +5,4 @@ spring:
   application:
     name: myth-executor-server
   profiles:
-    active: dev
+    active: local
\ No newline at end of file

byit-myth-executor/myth-executor-server/src/main/resources/banner.txt

+███╗   ███╗██╗   ██╗████████╗██╗  ██╗                   ██╗ ██████╗ ██████╗
+████╗ ████║╚██╗ ██╔╝╚══██╔══╝██║  ██║                   ██║██╔═══██╗██╔══██╗
+██╔████╔██║ ╚████╔╝    ██║   ███████║    █████╗         ██║██║   ██║██████╔╝
+██║╚██╔╝██║  ╚██╔╝     ██║   ██╔══██║    ╚════╝    ██   ██║██║   ██║██╔══██╗
+██║ ╚═╝ ██║   ██║      ██║   ██║  ██║              ╚█████╔╝╚██████╔╝██████╔╝
+╚═╝     ╚═╝   ╚═╝      ╚═╝   ╚═╝  ╚═╝               ╚════╝  ╚═════╝ ╚═════╝

byit-myth-gateway/src/main/resources/application-local.yml

+gateway:
+  load:
+    balance: ROUND
+logging:
+  config: classpath:logback.xml
+  level:
+    com:
+      byit:
+        gateway: DEBUG
+    org:
+      springframework: INFO
+myth-rpc:
+  registry:
+    address: http://localhost:8080/myth-register
+    biz: byit-myth-job
+    env: huangfu
+  remoting:
+    port: 8082
+
+spring:
+  datasource:
+    driver-class-name: com.mysql.jdbc.Driver
+    hikari:
+      idle-timeout: 60000
+      maximum-pool-size: 5
+      minimum-idle: 1
+    password: 123456
+    url: jdbc:mysql://10.0.10.118:3306/myth-registry?useUnicode=true&useSSL=true&characterEncoding=utf-8&mysqlEncoding=utf8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=false&autoReconnect=true&failOverReadOnly=false
+    username: root
+  zuul:
+    route:
+      refreshCron: 0/10 * * * * ?
+
+zuul:
+  ignored-services: '*'

byit-myth-gateway/src/main/resources/application.yml

@@ -4,4 +4,4 @@ spring:
   application:
     name: byit-myth-gateway
   profiles:
-    active: dev
+    active: local

byit-myth-gateway/src/main/resources/banner.txt

+███╗   ███╗██╗   ██╗████████╗██╗  ██╗                   ██╗ ██████╗ ██████╗
+████╗ ████║╚██╗ ██╔╝╚══██╔══╝██║  ██║                   ██║██╔═══██╗██╔══██╗
+██╔████╔██║ ╚████╔╝    ██║   ███████║    █████╗         ██║██║   ██║██████╔╝
+██║╚██╔╝██║  ╚██╔╝     ██║   ██╔══██║    ╚════╝    ██   ██║██║   ██║██╔══██╗
+██║ ╚═╝ ██║   ██║      ██║   ██║  ██║              ╚█████╔╝╚██████╔╝██████╔╝
+╚═╝     ╚═╝   ╚═╝      ╚═╝   ╚═╝  ╚═╝               ╚════╝  ╚═════╝ ╚═════╝

byit-myth-register/myth-register-server/src/main/resources/application-dev.yml

@@ -2,7 +2,7 @@ byit:
   registry:
     accessToken: ''
     data:
-      filepath: /Users/liyuan/codes/open/rpc/data/applogs/myth-registry/registrydata
+      filepath: /Users/dev/codes/open/rpc/data/applogs/myth-registry/registrydata
     login:
       password: 123456
       username: admin
@@ -14,13 +14,13 @@ mybatis:
 spring:
   datasource:
     driver-class-name: com.mysql.jdbc.Driver
-    password: 123456
+    password: root
     tomcat:
       max-active: 30
       max-wait: 10000
       test-on-borrow: true
     type: org.apache.tomcat.jdbc.pool.DataSource
-    url: jdbc:mysql://10.0.10.118:3306/myth-registry?Unicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false
+    url: jdbc:mysql://10.0.120.30:3307/myth-registry?Unicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false
     username: root
   freemarker:
     charset: UTF-8

byit-myth-register/myth-register-server/src/main/resources/application-local.yml

+byit:
+  registry:
+    accessToken: ''
+    data:
+      filepath: /Users/local/codes/open/rpc/data/applogs/myth-registry/registrydata
+    login:
+      password: 123456
+      username: admin
+logging:
+  config: classpath:logback.xml
+mybatis:
+  mapper-locations: classpath:/mybatis-mapper/*Mapper.xml
+
+spring:
+  datasource:
+    driver-class-name: com.mysql.jdbc.Driver
+    password: 123456
+    tomcat:
+      max-active: 30
+      max-wait: 10000
+      test-on-borrow: true
+    type: org.apache.tomcat.jdbc.pool.DataSource
+    url: jdbc:mysql://10.0.10.118:3306/myth-registry?Unicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false
+    username: root
+  freemarker:
+    charset: UTF-8
+    request-context-attribute: request
+    settings:
+      number_format: 0.##########
+    suffix: .ftl
+    templateLoaderPath: classpath:/templates/
+  mvc:
+    static-path-pattern: /static/**
+  resources:
+    static-locations: classpath:/static/

byit-myth-register/myth-register-server/src/main/resources/application.yml

@@ -4,4 +4,4 @@ server:
 
 spring:
   profiles:
-    active: dev
+    active: local
\ No newline at end of file

byit-myth-register/myth-register-server/src/main/resources/banner.txt

+███╗   ███╗██╗   ██╗████████╗██╗  ██╗                   ██╗ ██████╗ ██████╗
+████╗ ████║╚██╗ ██╔╝╚══██╔══╝██║  ██║                   ██║██╔═══██╗██╔══██╗
+██╔████╔██║ ╚████╔╝    ██║   ███████║    █████╗         ██║██║   ██║██████╔╝
+██║╚██╔╝██║  ╚██╔╝     ██║   ██╔══██║    ╚════╝    ██   ██║██║   ██║██╔══██╗
+██║ ╚═╝ ██║   ██║      ██║   ██║  ██║              ╚█████╔╝╚██████╔╝██████╔╝
+╚═╝     ╚═╝   ╚═╝      ╚═╝   ╚═╝  ╚═╝               ╚════╝  ╚═════╝ ╚═════╝

demo-client/byit-demo-client/src/main/java/com/byit/job/AddComplexPy.java

@@ -16,6 +16,8 @@ public class AddComplexPy {
         PluginPackage pluginPackage = new PluginPackage();
         pluginPackage.setWorkspaceName("test");
         pluginPackage.setFlow(createFlow());
+        JobUtils.setRequestUrl("http://127.0.0.1:8998");
+        JobUtils.setTOKEN("test");
         JobUtils.publish(pluginPackage);
     }
 

demo-client/byit-demo-client/src/main/java/com/byit/job/AddComplexPy1.java

+package com.byit.job;
+
+import com.alibaba.fastjson.JSON;
+import com.byit.dto.executor.ScriptParamAndPlaceholderDto;
+import com.byit.dto.plugin.*;
+import com.byit.utils.JobUtils;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author huanfgu
+ */
+public class AddComplexPy1 {
+    public static void main(String[] args) {
+        PluginPackage pluginPackage = new PluginPackage();
+        pluginPackage.setWorkspaceName("test");
+        pluginPackage.setFlow(createFlow());
+        JobUtils.setRequestUrl("http://127.0.0.1:8081");
+        JobUtils.setTOKEN("test");
+        JobUtils.publish(pluginPackage);
+    }
+
+
+    public static PluginFlow createFlow(){
+        PluginFlow pluginFlow = new PluginFlow();
+        //构建工作流信息
+        PluginFlowConfig build = PluginFlowConfig.builder().alarmEmail("huangfukexing@byitgroup.com")
+                .alarmlAction("1")
+                .execType("1")
+                .flowCron("0 0/1 * * * ? *")
+                .flowTimeout(TimeUnit.MINUTES.toMillis(30))
+                .priority("2")
+                .repeatCount(1)
+                .scheduleFollow("1")
+                .build();
+
+        pluginFlow.setName("复杂工作流");
+        pluginFlow.setDesc("测试多脚本复杂工作流创建");
+        pluginFlow.setConfig(build);
+        pluginFlow.setPrincipal("皇甫科星");
+        pluginFlow.setRePublish(false);
+        pluginFlow.setAuthor("huangfukexing");
+        pluginFlow.setNodeList(createNodes());
+        return pluginFlow;
+    }
+
+    /**
+     * 创建节点
+     * @return
+     */
+    public static List<PluginBaseNode> createNodes(){
+        //构建开始节点
+        PluginNode startNode = new PluginNode();
+        PluginNodeConfig startConf = new PluginNodeConfig();
+        nodeSet(startNode,false);
+        startNode.setName("start");
+        startNode.setDesc("开始节点");
+        startNode.setScriptUrls("ddmp/M00/00/01/CgB4Al5wa6WAfhQ9AAAAN9Xm1mU1221.py");
+        confSet(startConf);
+        startNode.setConfig(startConf);
+        System.out.println("-----------------start 节点构建成功,开始构建node1节点,依赖start节点--------------------");
+        PluginNode node1 = new PluginNode();
+        PluginNodeConfig node1Conf = new PluginNodeConfig();
+        nodeSet(node1,true);
+        node1.setName("node1");
+        node1.setDesc("node1节点");
+        node1.setScriptUrls("ddmp/M00/00/00/CgB4Al5wa36AaUJ7AAAAiYy1k-k9801.py");
+        confSet(node1Conf);
+        node1.setConfig(node1Conf);
+        node1.setDependNodeNameList(Collections.singletonList("start"));
+        System.out.println("-----------------node1 节点构建成功,开始构建 end,node1--------------------");
+        PluginNode endNode = new PluginNode();
+        PluginNodeConfig endNodeConf = new PluginNodeConfig();
+        nodeSet(endNode,false);
+        endNode.setName("end");
+        endNode.setDesc("end节点");
+        endNode.setScriptUrls("ddmp/M00/00/01/CgB4Al5wa6WADyyrAAAANb42CIc7700.py");
+        confSet(endNodeConf);
+        endNode.setConfig(endNodeConf);
+        endNode.setDependNodeNameList(Collections.singletonList("node1"));
+
+        return Arrays.asList(startNode,node1,endNode);
+    }
+
+
+    /**
+     * 构建节点
+     * @param pluginNode
+     */
+    private static void nodeSet(PluginNode pluginNode,boolean flag){
+        pluginNode.setAuthor("皇甫");
+        pluginNode.setJobType("SCRIPT");
+        pluginNode.setType("node");
+        pluginNode.setRunCommand("python ${biz_file}");
+
+        if(flag){
+            ScriptParamAndPlaceholderDto scriptParamAndPlaceholderDto = new ScriptParamAndPlaceholderDto();
+            Map<String,String> map = new HashMap<>();
+            map.put("name","皇甫科星");
+            scriptParamAndPlaceholderDto.setPlaceholder(map);
+            pluginNode.setRunParam(JSON.toJSONString(scriptParamAndPlaceholderDto));
+        }
+
+
+    }
+
+    /**
+     * 构建配置
+     * @param conf
+     */
+    private static void confSet( PluginNodeConfig conf){
+        conf.setFailedRetryCount(2);
+        conf.setFailedRetryInterval(TimeUnit.MINUTES.toSeconds(2));
+        conf.setNodeTimeout(-1L);
+        conf.setPriority("1");
+    }
+}