From 9a8b55a6743500789020528402000be45b423789 Mon Sep 17 00:00:00 2001 From: chenlw <874313221@qq.com> Date: Thu, 22 Sep 2016 14:34:23 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/com/base/CustomException.java | 5 +++- src/com/dao/mapper/data-details-mapper.xml | 4 +-- .../controller/DataModelController.java | 19 +++++++----- .../entities/OracleConnectorParams.java | 16 ++++++++++ src/com/platform/glusterfs/ShowData.java | 11 +++---- src/com/platform/glusterfs/VolumeInfo.java | 16 +++++----- src/com/platform/oracle/OracleConnector.java | 29 +++++++++++++------ src/com/platform/service/ILogRead.java | 7 +++++ .../platform/service/OracleExtractHelper.java | 20 ++++++------- .../service/impl/LogReadServiceImpl.java | 20 +++++++++++++ .../impl/OracleExtractServiceImpl.java | 10 +++---- .../service/thread/ThreadMoveData.java | 10 +++++-- src/com/platform/utils/FileOperateHelper.java | 3 ++ 13 files changed, 116 insertions(+), 54 deletions(-) create mode 100644 src/com/platform/service/ILogRead.java create mode 100644 src/com/platform/service/impl/LogReadServiceImpl.java diff --git a/src/com/base/CustomException.java b/src/com/base/CustomException.java index 3a5663c2..e59bb6bd 100644 --- a/src/com/base/CustomException.java +++ b/src/com/base/CustomException.java @@ -4,6 +4,9 @@ package com.base; import org.apache.log4j.Logger; +import com.platform.controller.DataModelController; +import com.platform.utils.Configs; + /** * <一句话功能简述> * <功能详细描述> @@ -16,7 +19,7 @@ import org.apache.log4j.Logger; public class CustomException extends Exception { /** log4j */ - public static Logger log = Logger.getRootLogger(); + public static Logger log = Configs.DAILY_ROLLING_LOGGER; /** 自定义异常信息-错误信息 */ private String msg; diff --git a/src/com/dao/mapper/data-details-mapper.xml b/src/com/dao/mapper/data-details-mapper.xml index bf5cf960..71df0de5 100644 --- a/src/com/dao/mapper/data-details-mapper.xml +++ b/src/com/dao/mapper/data-details-mapper.xml @@ -259,9 +259,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" data_info set extract_status = #{extractStatus} - regionalism_code = #{regionalismCode} - AND system_code = #{systemCode} - AND data_version = #{dataVersion} + id = #{id} AND remove ='0' diff --git a/src/com/platform/controller/DataModelController.java b/src/com/platform/controller/DataModelController.java index 7fe91f31..e8d2e979 100644 --- a/src/com/platform/controller/DataModelController.java +++ b/src/com/platform/controller/DataModelController.java @@ -33,6 +33,7 @@ import com.platform.entities.volumeMoveForm; import com.platform.service.DataInfoService; import com.platform.service.ICodeService; import com.platform.service.IGfsService; +import com.platform.service.ILogRead; import com.platform.service.IMoveDataService; import com.platform.service.IMySqlService; import com.platform.service.IOracleExtractService; @@ -43,7 +44,7 @@ import com.platform.utils.UtilsHelper; @Controller public class DataModelController extends BaseController{ - public static Logger log = Logger.getLogger(DataModelController.class); + public static Logger log = Configs.DAILY_ROLLING_LOGGER; @Resource(name = "dataInfoService") private DataInfoService dfs; @@ -63,6 +64,9 @@ public class DataModelController extends BaseController{ @Resource(name = "codeService") private ICodeService codeService; + @Resource(name = "logReadService") + private ILogRead logReadService; + public void setDfsImp(DataInfoService dfs) { this.dfs = dfs; } @@ -267,12 +271,13 @@ public class DataModelController extends BaseController{ @RequestMapping(value = "/oracle/extract/log", method= RequestMethod.POST ) @ResponseBody - public Object getExtractLog(@RequestParam("rcName") String name, HttpServletRequest res, HttpServletResponse req){ + public Object getExtractLog(@RequestParam("rcName") String name, HttpServletRequest res, HttpServletResponse req) throws Exception{ log.info("---------/oracle/extract/log-------------------"); - StringBuilder sb = new StringBuilder(); - sb.append("查看相应日志").append("\n").append("看到了"); - Map log = new HashMap<>(); - log.put(name, sb.toString()); - return log; + String result = logReadService.readLog(name); +// StringBuilder sb = new StringBuilder(); +// sb.append("查看相应日志").append("\n").append("看到了"); +// Map log = new HashMap<>(); +// log.put(name, sb.toString()); + return result; } } diff --git a/src/com/platform/entities/OracleConnectorParams.java b/src/com/platform/entities/OracleConnectorParams.java index 8391a84a..7fcefd5e 100644 --- a/src/com/platform/entities/OracleConnectorParams.java +++ b/src/com/platform/entities/OracleConnectorParams.java @@ -6,6 +6,8 @@ import com.platform.utils.ConfigPropertyReader; import com.platform.utils.Configs; public class OracleConnectorParams { + + private String dataId; private String port; private String ip; private String name; @@ -43,6 +45,20 @@ public class OracleConnectorParams { this.databaseName = properties.getProperty("collect-service-name"); } + /** + * @return the dataId + */ + public String getDataId() { + return dataId; + } + + /** + * @param dataId the dataId to set + */ + public void setDataId(String dataId) { + this.dataId = dataId; + } + public String getPort() { return port; } diff --git a/src/com/platform/glusterfs/ShowData.java b/src/com/platform/glusterfs/ShowData.java index 0ca8b438..8eb694c2 100644 --- a/src/com/platform/glusterfs/ShowData.java +++ b/src/com/platform/glusterfs/ShowData.java @@ -26,7 +26,7 @@ public class ShowData { * @see [类、类#方法、类#成员] */ public Map showVolumeFiles(String volumeName){ - log.info("start show the data"); +// log.info("start show the data"); Map data_type=new HashMap(); /** * get mount point of volumeName @@ -45,14 +45,13 @@ public class ShowData { * @return */ public Map showFolderData(String folderName){ - log.info(" start get "+folderName+" data"); +// log.info(" start get "+folderName+" data"); Map data_type=new HashMap(); String command="ls -l "+folderName; /* -<<<<<<< HEAD * RunCommand runCommand=new RunCommand(); List * reStrings=runCommand.runCommandWait(command); */ @@ -86,7 +85,7 @@ public class ShowData { data_type.put(keyValue[8], keyValue[1]); } - log.info(" get "+folderName+" data successed"); +// log.info(" get "+folderName+" data successed"); return data_type; } @@ -98,7 +97,7 @@ public class ShowData { * @see [类、类#方法、类#成员] */ public long getFolderSize(String folderPath) { - log.info("get " + folderPath + " Size "); +// log.info("get " + folderPath + " Size "); String command = "du -k -d 0 "+folderPath+" | grep " + folderPath + "|awk \'{print $1}\'"; List reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(command); @@ -111,8 +110,6 @@ public class ShowData { return -1; } long size = Long.valueOf(reStrings.get(0)); - - return size; } diff --git a/src/com/platform/glusterfs/VolumeInfo.java b/src/com/platform/glusterfs/VolumeInfo.java index e2ece069..f3583102 100644 --- a/src/com/platform/glusterfs/VolumeInfo.java +++ b/src/com/platform/glusterfs/VolumeInfo.java @@ -37,7 +37,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public List showAllVolumeName() { - log.info("get volume name"); +// log.info("get volume name"); List volNames = new ArrayList(); /* @@ -84,7 +84,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public String getVolumeType(String volumeName) { - log.info("get volume type"); +// log.info("get volume type"); String volType = ""; List reStrings = Constant.ganymedSSH @@ -123,7 +123,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public String getVolumeStatus(String volumeName) { - log.info("get volume status"); +// log.info("get volume status"); String volStatus = ""; String cmd = Constant.glusterVolumeInfo + " " + volumeName + " |grep ^Status"; List reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); @@ -162,7 +162,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public Long getVolumeAvailableSize(String volumeName) throws Exception{ - log.info("get volume availableSize"); +// log.info("get volume availableSize"); Long allSize = 0L; String cmd = Constant.df + " | grep " + volumeName + "|awk '{print $4}'"; @@ -188,7 +188,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public Long getVolumeUseSize(String volumeName) throws Exception{ - log.info("get volume used size"); +// log.info("get volume used size"); Long usedSize = 0L; if (volumeIsExists(volumeName) == false) { log.error("1901 " + volumeName + " is not exists!"); @@ -219,7 +219,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public List getVolumeBricks(String volumeName) { - log.info("get volume bricks"); +// log.info("get volume bricks"); String cmd = "gluster volume info " + volumeName + " |grep ^Brick'[0-9]\\+' |awk '{print $2}'"; List reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); @@ -243,7 +243,7 @@ public class VolumeInfo { * @see [类、类#方法、类#成员] */ public List getVolumeMountPoint(String volumeName) { - log.info("get volume MountPoint"); +// log.info("get volume MountPoint"); List mountPoints = new ArrayList<>(); String cmd=Constant.df + "|grep " + volumeName + "|awk '{print $6}'"; List reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); @@ -265,7 +265,7 @@ public class VolumeInfo { } public String getOneVolumeMountPoint(String volumeName) { - log.info("get one volume MountPoint"); +// log.info("get one volume MountPoint"); String mountPoint=null; diff --git a/src/com/platform/oracle/OracleConnector.java b/src/com/platform/oracle/OracleConnector.java index 5bff5eea..41978349 100644 --- a/src/com/platform/oracle/OracleConnector.java +++ b/src/com/platform/oracle/OracleConnector.java @@ -9,6 +9,7 @@ import java.sql.Statement; import com.base.Custom4exception; import com.base.CustomException; import com.platform.utils.Configs; +import com.platform.utils.FileOperateHelper; public class OracleConnector { static { @@ -39,16 +40,18 @@ public class OracleConnector { return (null != ConnectionBuilder(url, user, password)); } - public synchronized static ResultSet getSQLExecResultSet(Connection conn, String sql) { + public synchronized static ResultSet getSQLExecResultSet(Connection conn, String sql, String filePath) { ResultSet resultSet = null; try { Statement statement = conn .createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE); resultSet = statement.executeQuery(sql); + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+"OK \r\n"); } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+e.getMessage()+"\r\n"); new CustomException(Custom4exception.OracleSQL_Except, e); } @@ -56,8 +59,8 @@ public class OracleConnector { } public synchronized static ResultSet getSQLExecResultSet(String url, String user, - String password, String sql) { - return getSQLExecResultSet(ConnectionBuilder(url, user, password), sql); + String password, String sql, String filePath) { + return getSQLExecResultSet(ConnectionBuilder(url, user, password), sql, filePath); } /** @@ -66,15 +69,18 @@ public class OracleConnector { * @param sql * @return true:执行的不返回集合数据的sql成功， 是否执行成功 */ - public synchronized static boolean execOracleSQL(Connection conn, String sql) { + public synchronized static boolean execOracleSQL(Connection conn, String sql, String filePath) { boolean flag = false; try { Statement statement = conn.createStatement(); statement.execute(sql); flag = true; + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+"OK \r\n"); } catch (SQLException e) { flag = false; - e.printStackTrace(); + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+e.getMessage()+"\r\n"); new CustomException(Custom4exception.OracleSQL_Except, e); } return flag; @@ -85,15 +91,20 @@ public class OracleConnector { * @param sql * @return true:执行结果大于1，即有数据 是否执行成功 */ - public synchronized static boolean execUpdateOracleSQL(Connection conn, String sql) { + public synchronized static boolean execUpdateOracleSQL(Connection conn, String sql, String filePath) { boolean flag = false; try { Statement statement = conn.createStatement(); if(statement.executeUpdate(sql) > 0) + { flag = true; + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+"OK \r\n"); + } } catch (SQLException e) { flag = false; - e.printStackTrace(); + FileOperateHelper + .fileWrite(filePath, sql+ "\r\n"+e.getMessage()+"\r\n"); new CustomException(Custom4exception.OracleSQL_Except, e, sql); } return flag; diff --git a/src/com/platform/service/ILogRead.java b/src/com/platform/service/ILogRead.java new file mode 100644 index 00000000..3b42fca3 --- /dev/null +++ b/src/com/platform/service/ILogRead.java @@ -0,0 +1,7 @@ +package com.platform.service; + +public interface ILogRead { + + public String readLog(String filename) throws Exception; + +} diff --git a/src/com/platform/service/OracleExtractHelper.java b/src/com/platform/service/OracleExtractHelper.java index 46195223..1dcbdaad 100644 --- a/src/com/platform/service/OracleExtractHelper.java +++ b/src/com/platform/service/OracleExtractHelper.java @@ -27,7 +27,7 @@ public class OracleExtractHelper { String sql = "SELECT * FROM ALL_DB_LINKS WHERE DB_LINK='" + linkName+"'"; ResultSet rSet = null; try { - rSet = OracleConnector.getSQLExecResultSet(conn, sql); + rSet = OracleConnector.getSQLExecResultSet(conn, sql, null); rSet.last(); if (rSet.getRow() > 0) flag = true; @@ -66,10 +66,10 @@ public class OracleExtractHelper { .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), deleteSql+"\r\n"); - if (OracleConnector.execOracleSQL(conn, deleteSql)) { + if (OracleConnector.execOracleSQL(conn, deleteSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) { FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql+"\r\n"); - OracleConnector.execOracleSQL(conn, sql); + OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); } else { Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!"); FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION @@ -78,7 +78,7 @@ public class OracleExtractHelper { } else { // 否则，创建dblink - OracleConnector.execOracleSQL(conn, sql); + OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); FileOperateHelper.fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql); } @@ -95,7 +95,7 @@ public class OracleExtractHelper { public boolean createTableSpace(Connection conn, GatherOracleInfo oc) { String tmpSql = "select TABLESPACE_NAME from dba_tablespaces where TABLESPACE_NAME = '" + oc.getTableName() + "'"; - if (OracleConnector.execUpdateOracleSQL(conn, tmpSql)) { + if (OracleConnector.execUpdateOracleSQL(conn, tmpSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) { FileOperateHelper .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), @@ -110,7 +110,7 @@ public class OracleExtractHelper { .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql+"\r\n"); - return OracleConnector.execOracleSQL(conn, sql); + return OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); } } @@ -127,13 +127,13 @@ public class OracleExtractHelper { + Configs.GATHER_TABLE_PASSWORD; String grantSql = "grant connect, resource, dba to " + strTUser; - OracleConnector.execOracleSQL(conn, sql); + OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); FileOperateHelper .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql+"\r\n"); - OracleConnector.execOracleSQL(conn, grantSql); + OracleConnector.execOracleSQL(conn, grantSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); FileOperateHelper .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), @@ -158,7 +158,7 @@ public class OracleExtractHelper { + " t WHERE t.TABLESPACE_NAME NOT IN ('SYSTEM','SYSAUX')" + " and t.owner||t.table_name not in (select owner||table_name from dba_tables@LINKTO" + oc.getName() + " where 'data_type'='CLOB')"; - ResultSet rsSet = OracleConnector.getSQLExecResultSet(conn, sql); + ResultSet rsSet = OracleConnector.getSQLExecResultSet(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); FileOperateHelper .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), @@ -168,7 +168,7 @@ public class OracleExtractHelper { try { rsSet.getRow(); String resultSql = rsSet.getString(1); - OracleConnector.execUpdateOracleSQL(conn, resultSql); + OracleConnector.execUpdateOracleSQL(conn, resultSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); FileOperateHelper .fileWrite( Configs.EXTRACT_LOG_LOCALTION + oc.getName(), diff --git a/src/com/platform/service/impl/LogReadServiceImpl.java b/src/com/platform/service/impl/LogReadServiceImpl.java new file mode 100644 index 00000000..b5f32fd5 --- /dev/null +++ b/src/com/platform/service/impl/LogReadServiceImpl.java @@ -0,0 +1,20 @@ +package com.platform.service.impl; + +import org.springframework.stereotype.Service; + +import com.platform.service.ILogRead; +import com.platform.utils.Configs; +import com.platform.utils.FileOperateHelper; + +@Service(value = "logReadService") +public class LogReadServiceImpl implements ILogRead { + + + @Override + public String readLog(String filename) throws Exception { + + String result = FileOperateHelper.fileReader(Configs.EXTRACT_LOG_LOCALTION+filename+".log"); + return result; + } + +} diff --git a/src/com/platform/service/impl/OracleExtractServiceImpl.java b/src/com/platform/service/impl/OracleExtractServiceImpl.java index 8137062a..13447f0f 100644 --- a/src/com/platform/service/impl/OracleExtractServiceImpl.java +++ b/src/com/platform/service/impl/OracleExtractServiceImpl.java @@ -53,12 +53,10 @@ public class OracleExtractServiceImpl implements IOracleExtractService { Connection conn = OracleConnector.ConnectionBuilder("jdbc:oracle:thin:@" + oracleModel.getIp() + ":" + oracleModel.getPort() + ":" + oracleModel.getDatabaseName(), oracleModel.getUser(), oracleModel.getPassword()); for (OracleConnectorParams collectOracle : datainfos) { - String[] str = collectOracle.getName().split("-"); - if(str.length == 3){ + + if(null != collectOracle.getDataId() && !"".equals(collectOracle.getDataId())){ DataInfoEntity data = new DataInfoEntity(); - data.setRegionalismCode(str[0]); - data.setSystemCode(Integer.valueOf(str[1])); - data.setDataVersion(Integer.valueOf(str[2])); + data.setId(Integer.valueOf(collectOracle.getDataId())); data.setExtractStatus(1); dataInfoDao.updateExtract(data); collectOracle.setName("J" + collectOracle.getName().replace("-", "_")); @@ -68,7 +66,7 @@ public class OracleExtractServiceImpl implements IOracleExtractService { oracleExtract.createUser(conn, collectOracle, oracleModel);//创建用户并授权 oracleExtract.extractColleDB(conn, collectOracle);//执行抽取 // client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "2"); //更新oracle汇总状态，0标示为未汇总，1标示汇总中，2标示汇总完成 - data.setExtractStatus(1); + data.setExtractStatus(2); dataInfoDao.updateExtract(data); } } diff --git a/src/com/platform/service/thread/ThreadMoveData.java b/src/com/platform/service/thread/ThreadMoveData.java index cedcc319..a178ff0a 100644 --- a/src/com/platform/service/thread/ThreadMoveData.java +++ b/src/com/platform/service/thread/ThreadMoveData.java @@ -83,6 +83,8 @@ public class ThreadMoveData{ DataInfoEntityMoveTmp dataMove = result.get(i); //如果拷贝进度超过20分钟未进行-- 判断为 迁移失败。 // "1" :正在上传，0：等待 迁移， 2：成功 3：失败 + long srcSizeTemp = 0; + long dstSize = 0; if ("1".equals(dataMove.getCompleteStatus())) { long nowTime = new Date().getTime(); long timelong = nowTime - DateForm.string2DateBysecond(dataMove.getLastTime()).getTime(); @@ -96,9 +98,9 @@ public class ThreadMoveData{ isNoMove = false; } // 查询大小：。//gfs 获取size， - long srcSize = show.getFolderSize(dataMove.getDataPath()); - srcSize = (long) (srcSize * 0.998); - long dstSize = show.getFolderSize(dataMove.getDstPath()); + srcSizeTemp = show.getFolderSize(dataMove.getDataPath()); + long srcSize = (long) (srcSizeTemp * 0.998); + dstSize = show.getFolderSize(dataMove.getDstPath()); if (srcSize < 0) { log.error(dataMove.getDataPath() + " : 路径不存在！ "); continue; @@ -119,6 +121,8 @@ public class ThreadMoveData{ if("1".equals(dataMove.getCompleteStatus()) && dataMove.getRate() > 0){ //传输完毕：进行校验 if (realRate == 100) { + int difSize = (int) (srcSizeTemp-dstSize); + Thread.sleep(10*difSize); //TODO 查看当前拷贝目录进程是否结束？ //TODO 进行MD5校验 int resl = check.checkoutMD5Folder(dataMove.getDataPath(), dataMove.getDstPath()); diff --git a/src/com/platform/utils/FileOperateHelper.java b/src/com/platform/utils/FileOperateHelper.java index 6773436c..13ddc041 100644 --- a/src/com/platform/utils/FileOperateHelper.java +++ b/src/com/platform/utils/FileOperateHelper.java @@ -25,6 +25,9 @@ public class FileOperateHelper { */ @SuppressWarnings("resource") public static void fileWrite(String path, String message) { + if (null == path || "".equals(path)) { + return; + } try { File file = new File(path); if (file.exists())