From cb3af772855296b0c3c3516de8525747e4883b85 Mon Sep 17 00:00:00 2001 From: chenlw <874313221@qq.com> Date: Wed, 14 Sep 2016 09:32:55 +0800 Subject: [PATCH] =?UTF-8?q?=E9=9B=86=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/mybatis-applicationConfig.xml | 4 +- .../config/spring-applicationContext.xml | 9 +- WebContent/WEB-INF/web.xml | 2 +- src/com/dao/mapper/data-details-mapper.xml | 102 ++++++++++- src/com/dao/mapper/dataInfoMoveTmpmapper.xml | 67 +++++--- .../controller/DataModelController.java | 78 ++++++--- src/com/platform/dao/DataInfoDao.java | 8 +- src/com/platform/dao/DataInfoMoveTmpDao.java | 3 +- src/com/platform/entities/Brick.java | 19 ++- .../entities/DataInfoEntityMoveTmp.java | 21 ++- src/com/platform/entities/FolderNode.java | 3 +- src/com/platform/entities/VolumeEntity.java | 6 +- src/com/platform/glusterfs/CopyData.java | 45 ++++- src/com/platform/glusterfs/ShowData.java | 26 +++ src/com/platform/service/DataInfoService.java | 6 +- .../platform/service/DataInfoServiceImp.java | 20 ++- .../platform/service/impl/GfsServiceImpl.java | 86 ++++++---- .../service/thread/ThreadMoveData.java | 158 ++++++++++++++++++ src/com/platform/utils/CacheTreeData.java | 20 +++ src/com/platform/utils/ConfigsLoader.java | 2 + src/com/platform/utils/Constant.java | 7 + src/com/platform/utils/ThreadMoveData.java | 31 ---- src/com/platform/utils/ThreadVolume.java | 87 ++++++++-- 23 files changed, 663 insertions(+), 147 deletions(-) create mode 100644 src/com/platform/service/thread/ThreadMoveData.java delete mode 100644 src/com/platform/utils/ThreadMoveData.java diff --git a/WebContent/WEB-INF/config/mybatis-applicationConfig.xml b/WebContent/WEB-INF/config/mybatis-applicationConfig.xml index 85138096..f1679b32 100644 --- a/WebContent/WEB-INF/config/mybatis-applicationConfig.xml +++ b/WebContent/WEB-INF/config/mybatis-applicationConfig.xml @@ -8,9 +8,11 @@ + - + + \ No newline at end of file diff --git a/WebContent/WEB-INF/config/spring-applicationContext.xml b/WebContent/WEB-INF/config/spring-applicationContext.xml index 33ffe0a4..6f2a35ac 100644 --- a/WebContent/WEB-INF/config/spring-applicationContext.xml +++ b/WebContent/WEB-INF/config/spring-applicationContext.xml @@ -3,6 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p" xmlns:tx="http://www.springframework.org/schema/tx" xmlns:aop="http://www.springframework.org/schema/aop" xmlns:context="http://www.springframework.org/schema/context" + xmlns:task="http://www.springframework.org/schema/task" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/tx @@ -10,7 +11,9 @@ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd http://www.springframework.org/schema/context - http://www.springframework.org/schema/context/spring-context-3.0.xsd"> + http://www.springframework.org/schema/context/spring-context-3.0.xsd + http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.0.xsd"> @@ -71,4 +74,8 @@ + + + + \ No newline at end of file diff --git a/WebContent/WEB-INF/web.xml b/WebContent/WEB-INF/web.xml index d9935a26..cfdc2ad5 100644 --- a/WebContent/WEB-INF/web.xml +++ b/WebContent/WEB-INF/web.xml @@ -18,7 +18,7 @@ CharacterEncodingFilter - /* + / diff --git a/src/com/dao/mapper/data-details-mapper.xml b/src/com/dao/mapper/data-details-mapper.xml index 52b98767..89b9f52e 100644 --- a/src/com/dao/mapper/data-details-mapper.xml +++ b/src/com/dao/mapper/data-details-mapper.xml @@ -79,6 +79,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" + ORDER BY data_details.id LIMIT #{PagerOptions.limit} @@ -101,8 +102,107 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" + + + INSERT INTO + data_info( + + + regionalism_code, + + + system_code, + + + data_type, + + + data_version, + + + submitted_batch, + + + data_path, + + + collection_time, + + + collector_name, + + + collector_contacts, + + + data_charset, + + + data_year + + + ) + VALUES( + + + #{regionalismCode}, + + + #{systemCode}, + + + #{dataType}, + + + #{dataVersion}, + + + #{submittedBatch}, + + + #{dataPath}, + + + #{collectingTime}, + + + #{collectorName}, + + + #{collectorContacts}, + + + #{charset}, + + + #{year} + + + ) + + + + + + UPDATE + data_info + set remove = '1' + where id in + + #{item} + + and remove ='0' + \ No newline at end of file diff --git a/src/com/dao/mapper/dataInfoMoveTmpmapper.xml b/src/com/dao/mapper/dataInfoMoveTmpmapper.xml index c0545560..7d1145d6 100644 --- a/src/com/dao/mapper/dataInfoMoveTmpmapper.xml +++ b/src/com/dao/mapper/dataInfoMoveTmpmapper.xml @@ -41,16 +41,26 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" jdbcType="VARCHAR" /> + + + regionalism_code,system_code,dst_path,lasttime + + - SELECT - a.id,a.regionalism_code,b.city_name,b.district_name, a.system_code,b.system_name,b.data_type,b.data_version,b.submitted_batch, - b.data_path,b.data_charset,b.collection_time,b.collector_name,b.collector_contacts,b.data_year,a.dst_path,a.complete_status,a.rate + a.id id,a.regionalism_code regionalismCode,b.city_name cityName,b.district_name districtName, + a.system_code systemCode,b.system_name systemName,b.data_type dataType,b.data_version dataVersion, + b.submitted_batch submittedBatch,b.data_path dataPath,b.data_charset charset,b.collection_time collectionTime, + b.collector_name collectorName,b.collector_contacts collectorContacts,b.data_year dataYear,a.dst_path dstPath, + a.complete_status completeStatus,a.rate rate, a.lasttime lastTime FROM move_data_tmp a LEFT JOIN data_details b - ON a.system_code = b.system_code AND a.regionalism_code = b.regionalism_code; + ON a.system_code = b.system_code AND a.regionalism_code = b.regionalism_code + ORDER BY a.id @@ -58,19 +68,18 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" move_data_tmp - - regionalism_code = #{regionalismCode}, - - - system_code= #{systemCode}, - dst_path= #{dstPath}, complete_status= #{completeStatus}, - rate= #{rate} + + rate= #{rate}, + + + lasttime= #{lastTime}, + @@ -78,7 +87,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" - + INSERT INTO move_data_tmp( @@ -88,7 +97,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" regionalism_code, - + system_code, @@ -97,7 +106,12 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" complete_status, - rate + + rate, + + + lasttime, + ) VALUES( @@ -117,19 +131,32 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" #{completeStatus}, - #{rate} + + #{rate} + + + #{lastTime}, + ) - + + + DELETE FROM move_data_tmp WHERE id = #{id} - - + + + 获取数据查询的起始di + --> \ No newline at end of file diff --git a/src/com/platform/controller/DataModelController.java b/src/com/platform/controller/DataModelController.java index 1a2a09ed..6952659f 100644 --- a/src/com/platform/controller/DataModelController.java +++ b/src/com/platform/controller/DataModelController.java @@ -1,5 +1,6 @@ package com.platform.controller; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -24,6 +25,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import com.base.BaseController; import com.platform.entities.DataInfoEntity; +import com.platform.entities.DataInfoEntityMoveTmp; import com.platform.entities.FolderNode; import com.platform.entities.GatherOracleInfo; import com.platform.entities.OracleConnectorParams; @@ -31,6 +33,7 @@ import com.platform.entities.PagerOptions; import com.platform.entities.VolumeEntity; import com.platform.service.DataInfoService; import com.platform.service.IGfsService; +import com.platform.service.IMoveDataService; import com.platform.service.IMySqlService; import com.platform.service.IOracleExtractService; import com.platform.service.OracleExtractHelper; @@ -39,6 +42,7 @@ import com.platform.service.impl.MySqlServiceImpl; import com.platform.test.Brick; import com.platform.test.FolderReader; import com.platform.test.Volume; +import com.platform.utils.Bean2MapUtils; import com.platform.utils.Configs; import com.platform.utils.UtilsHelper; @@ -55,6 +59,9 @@ public class DataModelController extends BaseController{ @Resource(name = "OracleExtract") private IOracleExtractService OracleExtract; + + @Resource(name = "moveDataService") + private IMoveDataService moveDataService; public void setDfsImp(DataInfoService dfs) { this.dfs = dfs; @@ -63,11 +70,12 @@ public class DataModelController extends BaseController{ @RequestMapping("/data.json") @ResponseBody public ModelMap getAllDataToJson(HttpServletRequest res, - HttpServletResponse req) { + HttpServletResponse req) throws UnsupportedEncodingException { + res.setCharacterEncoding("UTF-8"); Map paramMap = res.getParameterMap(); Set keySet = paramMap.keySet(); Map params = new HashMap(); - StringBuffer sb = new StringBuffer().append("��ǰ���������:{"); + StringBuffer sb = new StringBuffer().append("当前的请求参数:{"); for (String str : keySet) { String value = paramMap.get(str)[0]; if (StringUtils.isNotEmpty(value)) { @@ -81,19 +89,21 @@ public class DataModelController extends BaseController{ .append("}").toString()); PagerOptions pagerOptions = (PagerOptions) UtilsHelper .newObjAndSetAttrsByClass(PagerOptions.class, params); - + return dfs.getPagerTableData(pagerOptions); } - @RequestMapping("/delete/data") - public void deleteData(HttpServletRequest res, HttpServletResponse req) { + @RequestMapping(value="/delete/data", method= RequestMethod.POST) + public void deleteData(HttpServletRequest res, HttpServletResponse req) throws Exception { + res.setCharacterEncoding("UTF-8"); Map paramMap = res.getParameterMap(); String[] data = paramMap.get("data"); dfs.deleteData(data); } @RequestMapping("/connectOracle") - public void connectOracle(HttpServletRequest res, HttpServletResponse req) { + public void connectOracle(HttpServletRequest res, HttpServletResponse req) throws UnsupportedEncodingException { + res.setCharacterEncoding("UTF-8"); Map paramMap = res.getParameterMap(); String[] oraclesName = paramMap.get("oracleName"); if (oraclesName != null) @@ -105,21 +115,23 @@ public class DataModelController extends BaseController{ @RequestMapping("/cancelOracleConection") public void cancelOracleConnection(HttpServletRequest res, - HttpServletResponse req) { + HttpServletResponse req) throws UnsupportedEncodingException { + res.setCharacterEncoding("UTF-8"); Map paramMap = res.getParameterMap(); String[] oraclesName = paramMap.get("oracleName"); String operate = paramMap.get("operation")[0]; if (null != oraclesName) { for (String rcName : oraclesName) { - Configs.CONSOLE_LOGGER.info("连接成功:\t" + rcName); + Configs.CONSOLE_LOGGER.info("取消连接:\t" + rcName); new OracleStatusService().cancelToOracle(rcName, operate); } } } - @RequestMapping("/oracle/{name}/extract") + @RequestMapping(value="/oracle/{name}/extract", method= RequestMethod.POST) public void extractOracleData(HttpServletRequest res, HttpServletResponse req, String name) throws Exception { + res.setCharacterEncoding("UTF-8"); System.out.println("------extract-------"); System.out.println(name); Map paramMap = res.getParameterMap(); @@ -144,7 +156,7 @@ public class DataModelController extends BaseController{ } - @RequestMapping("/volume/list") + @RequestMapping(value="/volume/list", method= RequestMethod.POST) @ResponseBody public List getFolder(HttpServletRequest res, HttpServletResponse req) throws Exception { System.out.println("get Request"); @@ -161,6 +173,7 @@ public class DataModelController extends BaseController{ @ResponseBody public Object move(HttpServletRequest res, HttpServletResponse req, String name, FolderNode selectNode, DataInfoEntity selectItems) throws Exception { + res.setCharacterEncoding("UTF-8"); System.out.println("get Request"); Map paramMap = res.getParameterMap(); // System.out.println(paramMap); @@ -174,22 +187,23 @@ public class DataModelController extends BaseController{ List listItemPath = new ArrayList(); String[] items = paramMap.get("selectItems"); System.out.println("selectItems"); + List datas = new ArrayList(); for (String string : items) { System.out.println(string); JSONObject jsobj = JSONObject.fromObject(string); Map itemmap = jsobj; - listItemPath.add((String)itemmap.get("dataPath")); + DataInfoEntity data = (DataInfoEntity) Bean2MapUtils.convertMap(DataInfoEntity.class, itemmap); + datas.add(data); + } + System.out.println("------/volume/{name}/move--------"); + boolean result = false ; + if (datas.size() >0) { + result = moveDataService.moveData(datas, (String)map.get("path")); } - - System.out.println(name); - System.out.println(nodes); - System.out.println(selectItems); - System.out.println("--------------"); - int result = gfsService.copyFolder(listItemPath, (String)map.get("path"), "app"); return result; } - @RequestMapping("/oracle/list") + @RequestMapping(value="/oracle/list", method= RequestMethod.POST) @ResponseBody public List getOracleInfo(HttpServletRequest res, HttpServletResponse req) throws Exception { @@ -199,30 +213,48 @@ public class DataModelController extends BaseController{ return result; } - @RequestMapping("/oracle/{id}/delete") + @RequestMapping(value="/oracle/{id}/delete", method= RequestMethod.POST) @ResponseBody public void deleteOracleInfo(HttpServletRequest res, HttpServletResponse req, int id) throws Exception { - + res.setCharacterEncoding("UTF-8"); mySqlService.deleteMySql(id); System.out.println("----------deleteOracleInfo-----------------------"); } - @RequestMapping("/oracle/{id}/insert") + @RequestMapping(value="/oracle/{id}/insert", method= RequestMethod.POST) @ResponseBody public void insertOracleInfo(HttpServletRequest res, HttpServletResponse req, GatherOracleInfo oracle) throws Exception { - + res.setCharacterEncoding("UTF-8"); mySqlService.insertOracle(oracle); System.out.println("----------insertOracleInfo-----------------------"); } - @RequestMapping("/oracle/{id}/update") + @RequestMapping(value="/oracle/update", method= RequestMethod.POST) @ResponseBody public void updateOracleInfo(HttpServletRequest res, HttpServletResponse req, GatherOracleInfo oracle) throws Exception { + res.setCharacterEncoding("gb2312"); + System.out.println(oracle.getName()); + System.out.println(oracle); mySqlService.updateOracle(oracle); System.out.println("----------updateOracleInfo-----------------------"); } + + @RequestMapping(value="/task/transfer/list", method= RequestMethod.POST) + @ResponseBody + public Object taskList() throws Exception { + List result = moveDataService.findAll(); + return result; + } + + @RequestMapping(value="/task/transfer/deletes", method= RequestMethod.POST) + @ResponseBody + public Object taskdeletes() throws Exception { + DataInfoEntityMoveTmp dataMove = null; + int result = moveDataService.delete(dataMove); + return result; + } } diff --git a/src/com/platform/dao/DataInfoDao.java b/src/com/platform/dao/DataInfoDao.java index f42b9f8f..f8423548 100644 --- a/src/com/platform/dao/DataInfoDao.java +++ b/src/com/platform/dao/DataInfoDao.java @@ -3,10 +3,12 @@ package com.platform.dao; import java.util.List; import org.apache.ibatis.annotations.Param; +import org.springframework.stereotype.Repository; import com.platform.entities.DataInfoEntity; import com.platform.entities.PagerOptions; +@Repository(value = "dataInfoDao") public interface DataInfoDao { int getLimitedDataCount(@Param("PagerOptions")PagerOptions pagerOptions); @@ -15,6 +17,10 @@ public interface DataInfoDao { List getLimitedDataInfoEntities(@Param("PagerOptions")PagerOptions pagerOptions); - List getIdIsExist(int parseInt); + List getIdIsExist(List list)throws Exception; + + int removes(List list)throws Exception; + + int save(DataInfoEntity data) throws Exception; } diff --git a/src/com/platform/dao/DataInfoMoveTmpDao.java b/src/com/platform/dao/DataInfoMoveTmpDao.java index 6d6a1d6a..096d8938 100644 --- a/src/com/platform/dao/DataInfoMoveTmpDao.java +++ b/src/com/platform/dao/DataInfoMoveTmpDao.java @@ -23,6 +23,7 @@ public interface DataInfoMoveTmpDao { int save(DataInfoEntityMoveTmp data) throws Exception; - int remove(DataInfoEntityMoveTmp data) throws Exception; + void insertBatch(List list) throws Exception; + int remove(int id) throws Exception; } diff --git a/src/com/platform/entities/Brick.java b/src/com/platform/entities/Brick.java index 06a8e9c4..53337ac9 100644 --- a/src/com/platform/entities/Brick.java +++ b/src/com/platform/entities/Brick.java @@ -32,6 +32,11 @@ public class Brick { /** 路径 */ private String path; + + /** + * true 有连接， false： 失去连接 + */ + private boolean status; /** * @return the availableSize @@ -89,5 +94,17 @@ public class Brick { this.path = path; } - + /** + * @return the status + */ + public boolean isStatus() { + return status; + } + + /** + * @param status the status to set + */ + public void setStatus(boolean status) { + this.status = status; + } } diff --git a/src/com/platform/entities/DataInfoEntityMoveTmp.java b/src/com/platform/entities/DataInfoEntityMoveTmp.java index 74be9e5d..0fb469e5 100644 --- a/src/com/platform/entities/DataInfoEntityMoveTmp.java +++ b/src/com/platform/entities/DataInfoEntityMoveTmp.java @@ -7,6 +7,12 @@ public class DataInfoEntityMoveTmp extends DataInfoEntity { private String completeStatus; private int rate; + + private String lastTime; // 采集时间 + + public DataInfoEntityMoveTmp() { + // TODO Auto-generated constructor stub + } /** * @return the dstPath @@ -49,6 +55,19 @@ public class DataInfoEntityMoveTmp extends DataInfoEntity { public void setRate(int rate) { this.rate = rate; } - + + /** + * @return the lastTime + */ + public String getLastTime() { + return lastTime; + } + + /** + * @param lastTime the lastTime to set + */ + public void setLastTime(String lastTime) { + this.lastTime = lastTime; + } } diff --git a/src/com/platform/entities/FolderNode.java b/src/com/platform/entities/FolderNode.java index 8f584d77..9ec6e8f8 100644 --- a/src/com/platform/entities/FolderNode.java +++ b/src/com/platform/entities/FolderNode.java @@ -1,12 +1,13 @@ package com.platform.entities; +import java.util.ArrayList; import java.util.List; public class FolderNode { private String name; private int isFolder; // 1 is file and other integer is folder show children number private String path; - private List childNodes; + private List childNodes = new ArrayList(); public FolderNode() { // TODO Auto-generated constructor stub diff --git a/src/com/platform/entities/VolumeEntity.java b/src/com/platform/entities/VolumeEntity.java index 84455003..f66670d4 100644 --- a/src/com/platform/entities/VolumeEntity.java +++ b/src/com/platform/entities/VolumeEntity.java @@ -37,7 +37,7 @@ public class VolumeEntity { private String path; /** volume树形目录 */ - private FolderNode folder; + private List folder = new ArrayList(); /** volume的 块 */ private List brick = new ArrayList(); @@ -101,14 +101,14 @@ public class VolumeEntity { /** * @return the folder */ - public FolderNode getFolder() { + public List getFolder() { return folder; } /** * @param folder the folder to set */ - public void setFolder(FolderNode folder) { + public void setFolder(List folder) { this.folder = folder; } diff --git a/src/com/platform/glusterfs/CopyData.java b/src/com/platform/glusterfs/CopyData.java index 6fd1e357..6f9e7eba 100644 --- a/src/com/platform/glusterfs/CopyData.java +++ b/src/com/platform/glusterfs/CopyData.java @@ -1,8 +1,9 @@ package com.platform.glusterfs; import java.util.List; - import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; @@ -46,6 +47,13 @@ public class CopyData { */ public int copyFolderFiles(String sourceFolderName, String destFolderName, String fileName) { int progress=0; + // 末尾 含有 / + Pattern pattern2 = Pattern.compile("\\/$"); + Matcher matcher = pattern2.matcher(sourceFolderName); + if (matcher.find()) { + sourceFolderName = sourceFolderName.substring(0, sourceFolderName.length()-1); + } + log.info("start copy " + fileName + " from " + sourceFolderName + " to " + destFolderName); ShowData showData=new ShowData(); Map reStrings=showData.showFolderData(destFolderName); @@ -71,6 +79,41 @@ public class CopyData { return 1; } + /** + * -1 :error; -2: the filename is not exists ;-3 :destFolderName ; 1: right + * not exists + * + * @param folderName + * @param fileName + * @return + */ + public int copyFolderFiles(String sourceFolderName, String destFolderName) { + int progress=0; + log.info("start copy " + " from " + sourceFolderName + " to " + destFolderName); + ShowData showData=new ShowData(); + Map reStrings=showData.showFolderData(destFolderName); + if(reStrings==null){ + log.info("3201 "+destFolderName+" is not exists"); + return -3; + } + + reStrings=showData.showFolderData(sourceFolderName); + if(reStrings==null){ + log.info("3202 "+sourceFolderName+" is not exists"); + return -2; + } + String command = "cp -r " + sourceFolderName+" "+destFolderName; + /* + * RunCommand runCommand = new RunCommand(); + + List reStrings = runCommand.runCommandWait(command); + */ + Constant.ganymedSSH.execCmdNoWaitAcquiescent(command); + + log.info("copy " + sourceFolderName +"/" + " to " + destFolderName + " running"); + return 1; + } + @Test public void testCopyFolderFiles() { diff --git a/src/com/platform/glusterfs/ShowData.java b/src/com/platform/glusterfs/ShowData.java index 77bd9867..49481923 100644 --- a/src/com/platform/glusterfs/ShowData.java +++ b/src/com/platform/glusterfs/ShowData.java @@ -89,6 +89,32 @@ public class ShowData { return data_type; } + /** + * 返回folder的大小字节表示 + * -2表示获取大小出错，-1表示folder不存在，其他表示folder的大小 + * @param folderPath + * @return + * @see [类、类#方法、类#成员] + */ + public long getFolderSize(String folderPath) { + log.info("get " + folderPath + " Size "); + + String command = "du -k -d 0 "+folderPath+" | grep " + folderPath + "|awk \'{print $1}\'"; + List reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(command); + if(reStrings==null || reStrings.size()==0){ + log.error("get " + folderPath + " Size error!"); + return -2; + } + if (reStrings.get(0).contains(Constant.noSuchFile)) { + log.error(folderPath+" is not exists"); + return -1; + } + long size = Long.valueOf(reStrings.get(0)); + + + return size; + } + /** * diff --git a/src/com/platform/service/DataInfoService.java b/src/com/platform/service/DataInfoService.java index be0ca7f7..fb2c2886 100644 --- a/src/com/platform/service/DataInfoService.java +++ b/src/com/platform/service/DataInfoService.java @@ -2,10 +2,14 @@ package com.platform.service; import org.springframework.ui.ModelMap; +import com.platform.dao.DataInfoDao; +import com.platform.entities.DataInfoEntity; import com.platform.entities.PagerOptions; public interface DataInfoService { public ModelMap getPagerTableData(PagerOptions pagerOptions); - void deleteData(String[] id); + void deleteData(String[] id) throws Exception; + + int save(DataInfoEntity data) throws Exception; } diff --git a/src/com/platform/service/DataInfoServiceImp.java b/src/com/platform/service/DataInfoServiceImp.java index bcbe94fb..4da74444 100644 --- a/src/com/platform/service/DataInfoServiceImp.java +++ b/src/com/platform/service/DataInfoServiceImp.java @@ -1,5 +1,6 @@ package com.platform.service; +import java.util.ArrayList; import java.util.List; import javax.annotation.Resource; @@ -24,7 +25,8 @@ public class DataInfoServiceImp implements DataInfoService { public ModelMap getPagerTableData(PagerOptions pagerOptions) { // TODO Auto-generated method stub ModelMap modelMap = new ModelMap(); - int count = dfdDao.getLimitedDataCount(pagerOptions); //获取总记录条数 + int count = dfdDao.getLimitedDataCount(pagerOptions); //获取总记录条数 + System.out.println("total colume " + count); int offset = 0; if (pagerOptions.getCurrentPageNum() > 1) { pagerOptions.setTotalLimit((pagerOptions.getCurrentPageNum() - 1) @@ -41,19 +43,29 @@ public class DataInfoServiceImp implements DataInfoService { } @Override - public void deleteData(String[] id) { + public void deleteData(String[] id) throws Exception { // TODO Auto-generated method stub + List ids = new ArrayList(); for(String idx: id){ + ids.add(Integer.parseInt(idx)); + } + if (ids.size() > 0) { //数据在不在？ - List paths = dfdDao.getIdIsExist(Integer.parseInt(idx)); + List paths = dfdDao.getIdIsExist(ids); if(paths.size()>0){ //删除文件操作 for (int i = 0; i < paths.size(); i++) { System.out.println(paths.get(i)); } //删除数据库记录 - //dfdDao.deleteRow(idx); + dfdDao.removes(ids); } } } + + @Override + public int save(DataInfoEntity data) throws Exception { + int result = dfdDao.save(data); + return result; + } } diff --git a/src/com/platform/service/impl/GfsServiceImpl.java b/src/com/platform/service/impl/GfsServiceImpl.java index 683ac2f1..535b9870 100644 --- a/src/com/platform/service/impl/GfsServiceImpl.java +++ b/src/com/platform/service/impl/GfsServiceImpl.java @@ -71,46 +71,51 @@ public class GfsServiceImpl implements IGfsService { */ @Override public List getAllVolumes() throws Exception { - List volumeList = new ArrayList<>(); - List volumeNameList = volumeInfo.showAllVolumeName(); - if (null == volumeNameList) { - return null; + List volumeList = CacheTreeData.getVolumeList(); + if (null == volumeList) { + return new ArrayList(); } - for (String volumeName : volumeNameList) { - VolumeEntity volume = new VolumeEntity(); - volume.setName(volumeName); - List path = volumeInfo.getVolumeMountPoint(volumeName); - //默认加载第一个路径 - if (null != path && path.size() > 0) { - volume.setPath(path.get(0)); - } - volume.setAllSize(volumeInfo.getVolumeAllSize(volumeName)); - volume.setUsedSize(volumeInfo.getVolumeUseSize(volumeName)); - //TODO 查询brick-- - //返回 ip:path - List brickStrs = volumeInfo.getVolumeBricks(volumeName); - //brick已用大小： - Map usedSize = volumeInfo.getVolumebricksDataSize(volumeName); - Map availableSize = volumeInfo.getVolumebricksAvailableSize(volumeName); - List brickList = new ArrayList(); - for (String brickIpPath : brickStrs) { - Brick b = new Brick(); - String ipAndpath[] = brickIpPath.split(":"); - String brickip = ipAndpath[0]; - String brickpath = ipAndpath[1]; - //iP , path , - b.setIp(brickip); - b.setPath(brickpath); - b.setAvailableSize(availableSize.get(brickIpPath)); - b.setUsedSize(usedSize.get(brickIpPath)); - brickList.add(b); - } - volume.setBrick(brickList); + for (VolumeEntity volume : volumeList) { +// VolumeEntity volume = new VolumeEntity(); +// volume.setName(volumeName); +// List path = volumeInfo.getVolumeMountPoint(volumeName); +// //默认加载第一个路径 +// if (null != path && path.size() > 0) { +// volume.setPath(path.get(0)); +// } +// volume.setAllSize(volumeInfo.getVolumeAllSize(volumeName)); +// volume.setUsedSize(volumeInfo.getVolumeUseSize(volumeName)); +// //TODO 查询brick-- +// //返回 ip:path +// List brickStrs = volumeInfo.getVolumeBricks(volumeName); +// //brick已用大小： +// Map usedSize = volumeInfo.getVolumebricksDataSize(volumeName); +// Map availableSize = volumeInfo.getVolumebricksAvailableSize(volumeName); +// +// +// List brickList = new ArrayList(); +// for (String brickIpPath : brickStrs) { +// Brick b = new Brick(); +// String ipAndpath[] = brickIpPath.split(":"); +// String brickip = ipAndpath[0]; +// String brickpath = ipAndpath[1]; +// //iP , path , +// b.setIp(brickip); +// b.setPath(brickpath); +// b.setAvailableSize(availableSize.get(brickIpPath)); +// b.setUsedSize(usedSize.get(brickIpPath)); +// brickList.add(b); +// } +// volume.setBrick(brickList); if (null != volume.getPath()) { // 获得 folder 目录 - volume.setFolder(getFolder(volume.getPath())); + List list = new ArrayList(); + FolderNode currNode = getFolder(volume.getPath()); + if (null != currNode && null != currNode.getChildNodes()) { + list.addAll(currNode.getChildNodes()); + } + volume.setFolder(list); } - volumeList.add(volume); } return volumeList; } @@ -127,7 +132,14 @@ public class GfsServiceImpl implements IGfsService { volume.setUsedSize(volumeInfo.getVolumeUseSize(volumeName)); // volume.setBrick(brick); if (null != volume.getPath()) { - volume.setFolder(getFolder(path.get(0))); + List list = new ArrayList(); + if (null != path && path.size() > 0) { + FolderNode currNode = getFolder(path.get(0)); + if (null != currNode && null != currNode.getChildNodes()) { + list.addAll(currNode.getChildNodes()); + } + } + volume.setFolder(list); } return volume; } diff --git a/src/com/platform/service/thread/ThreadMoveData.java b/src/com/platform/service/thread/ThreadMoveData.java new file mode 100644 index 00000000..ae319d4c --- /dev/null +++ b/src/com/platform/service/thread/ThreadMoveData.java @@ -0,0 +1,158 @@ +package com.platform.service.thread; + +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Resource; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; + +import com.platform.dao.DataInfoDao; +import com.platform.dao.DataInfoMoveTmpDao; +import com.platform.dao.GatherOracleDao; +import com.platform.entities.DataInfoEntity; +import com.platform.entities.DataInfoEntityMoveTmp; +import com.platform.entities.GatherOracleInfo; +import com.platform.glusterfs.CheckoutMD5; +import com.platform.glusterfs.CopyData; +import com.platform.glusterfs.ShowData; +import com.platform.service.DataInfoService; +import com.platform.service.DataInfoServiceImp; +import com.platform.service.IMoveDataService; +import com.platform.service.impl.MoveDataServiceImpl; +import com.platform.utils.Bean2MapUtils; +import com.platform.utils.Constant; +import com.platform.utils.DateForm; + +@Component +public class ThreadMoveData{ + + @Resource(name = "dataInfoDao") + private DataInfoDao dataInfoDao; + + /** + * 迁移 + */ + CopyData copy = new CopyData(); + + /** + * MD5校验 + */ + CheckoutMD5 check = new CheckoutMD5(); + + @Resource(name = "dataInfoMoveTmpDao") + private DataInfoMoveTmpDao dataInfoMoveTmpDao; + + /** + * 查看数据 + */ + ShowData show = new ShowData(); + + /** + * : 实时更新数据库--根据查询到的 正则迁移的数据 + */ + public ThreadMoveData() { + } + + //5秒 + @Scheduled(fixedDelay = 5000) + public void doSomething() { + + try { + List result = null; + //查询 表 move_data_tmp + result = dataInfoMoveTmpDao.findAll(); + if (null != result) { + //gfs 获取size， + int rsize = result.size(); + boolean isNoMove = true; + //该循环必须 循环每个，不能有 break; + // rate:大小：假的，待换成真实比例 + double realRate = 0.00; + for (int i = 0; i < rsize; i++) { + DataInfoEntityMoveTmp dataMove = result.get(i); + //如果拷贝进度超过20分钟未进行-- 判断为 迁移失败。 + // "1" :正在上传，0：等待 迁移， 2：成功 3：失败 + if ("1".equals(dataMove.getCompleteStatus())) { + long nowTime = new Date().getTime(); + long timelong = nowTime - DateForm.string2DateBysecond(dataMove.getLastTime()).getTime(); + if (timelong > 1000*60*20) { + dataMove.setCompleteStatus("3"); + dataInfoMoveTmpDao.update(dataMove); + } + isNoMove = false; + // 查询大小：。//gfs 获取size， + long srcSize = show.getFolderSize(dataMove.getDataPath()); + long dstSize = show.getFolderSize(dataMove.getDstPath()); + if (srcSize > 0 && dstSize > 0) { + realRate = dstSize / srcSize * 100; + dataMove.setRate((int) realRate); + dataMove.setLastTime(DateForm.date2StringBysecond(new Date())); + } + + } + if("1".equals(dataMove.getCompleteStatus()) &&dataMove.getRate() > 0){ + //传输完毕：进行校验 + if (realRate == 100) { + //TODO 进行MD5校验 + int resl = check.checkoutMD5Folder(dataMove.getDataPath(), dataMove.getDstPath()); + //TODO 校验成功--则删除数据库记录 + if(resl == 1){ + //校验成功--修改 数据库记录-- + dataMove.setCompleteStatus("2"); + dataMove.setLastTime(DateForm.date2StringBysecond(new Date())); + dataInfoMoveTmpDao.update(dataMove); + //TODO 新增 一条数据-到-dataInfo + DataInfoEntity data = (DataInfoEntity) Bean2MapUtils.convertMap( + DataInfoEntity.class, Bean2MapUtils.convertBean(dataMove)); + data.setDataPath(dataMove.getDstPath()); + data.setYear(dataMove.getYear()+" (迁移完成时间 " + DateForm.date2StringByMin(new Date())+")"); + dataInfoDao.save(data); + } + else { + // 3:表示 迁移失败 + dataMove.setCompleteStatus("3"); + } + } + dataMove.setLastTime(DateForm.date2StringBysecond(new Date())); + dataInfoMoveTmpDao.update(dataMove); + } + } + //循环 完了, 确定没有上传的 ,没有正在上传的 + if (isNoMove) { + //查询 表 move_data_tmp + result = dataInfoMoveTmpDao.findAll(); + if (null != result) { + + int tmpsize = result.size(); + // 上传下一个后 + for (int i = 0; i < tmpsize; i++) { + DataInfoEntityMoveTmp next2move = result.get(i); + //如果是 待 迁移状态的 + if ("0".equals(next2move.getCompleteStatus())) { + //待迁移的数据 -- 开始迁移 + copy.copyFolderFiles(next2move.getDataPath(), next2move.getDstPath(), "app"); + // "1" :正在上传，0：等待 迁移， 2：成功 3：失败 + next2move.setCompleteStatus("1"); + next2move.setLastTime(DateForm.date2StringBysecond(new Date())); + //更新sql + dataInfoMoveTmpDao.update(next2move); + break; + } + } + } + } + } + + Thread.sleep(Constant.update_dataInfo_sleep_time); + } catch (Exception e) { + System.err.println(e); + } + + } + +} diff --git a/src/com/platform/utils/CacheTreeData.java b/src/com/platform/utils/CacheTreeData.java index e1c6eb69..742df9a2 100644 --- a/src/com/platform/utils/CacheTreeData.java +++ b/src/com/platform/utils/CacheTreeData.java @@ -1,12 +1,16 @@ package com.platform.utils; import java.util.List; +import java.util.Map; import com.platform.entities.FolderNode; +import com.platform.entities.VolumeEntity; public class CacheTreeData { private static List folders = null; + + private static List volumeList = null; public static List getFolders() { return folders; @@ -15,5 +19,21 @@ public class CacheTreeData { public static void setFolders(List folders) { CacheTreeData.folders = folders; } + + /** + * @return the volumeList + */ + public static List getVolumeList() { + return volumeList; + } + + /** + * @param volumeList the volumeList to set + */ + public static void setVolumeList(List volumeList) { + CacheTreeData.volumeList = volumeList; + } } + + diff --git a/src/com/platform/utils/ConfigsLoader.java b/src/com/platform/utils/ConfigsLoader.java index 56745ce0..ef260782 100644 --- a/src/com/platform/utils/ConfigsLoader.java +++ b/src/com/platform/utils/ConfigsLoader.java @@ -5,6 +5,8 @@ import java.util.Properties; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import com.platform.service.thread.ThreadMoveData; + public class ConfigsLoader implements ServletContextListener { private static ConfigPropertyReader cReader = null; diff --git a/src/com/platform/utils/Constant.java b/src/com/platform/utils/Constant.java index ac7e642f..aaaadb5c 100644 --- a/src/com/platform/utils/Constant.java +++ b/src/com/platform/utils/Constant.java @@ -12,13 +12,20 @@ public class Constant { public static String glusterVolumeInfo = "gluster volume info "; public static String df = "df -k "; public static String peerincluster_connected="PeerinCluster(Connected)"; + public static String peerincluster_disconnected="PeerinCluster(Disconnected)"; public static String distributed="distributed"; public static String replica="replica"; public static String stripe="stripe"; public static String noVolume="No volumes present"; + public static String noSuchFile = "No such file or directory"; public static GanymedSSH ganymedSSH=new GanymedSSH(hostIp, rootUser, rootPasswd, port); public static String fileGetTreeData="./WEB-INF/config/getTreedata.sh"; + public static String strGetTreeData = "function ergodic(){\n " + + "for file in \\`ls \\$1\\`\n do\n if [ \"\\$file\" != \"app\" -a -d \\$1\\\"/\\\"\\$file ]\n " + + "then\n ergodic \\$1\"/\"\\$file\n else\n local path=\\$1\"/\"\\$file\n " + + "echo \\$path \n fi\n done\n}\n\nIFS=\\$\\'\\n\\' " + + "#这个必须要，否则会在文件名中有空格时出错\nINIT_PATH=\".\";\nergodic \\$1\n"; /** * volume 获取的线程休眠时间 diff --git a/src/com/platform/utils/ThreadMoveData.java b/src/com/platform/utils/ThreadMoveData.java deleted file mode 100644 index e05c07b1..00000000 --- a/src/com/platform/utils/ThreadMoveData.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.platform.utils; - -public class ThreadMoveData extends Thread { - - /** - * : 实时更新数据库--根据查询到的 正则迁移的数据 - */ - public ThreadMoveData() { - // TODO Auto-generated constructor stub - } - - /* (non-Javadoc) - * @see java.lang.Thread#run() - * - */ - @Override - public void run() { - // TODO Auto-generated method stub - super.run(); - //查询 表 move_data_tmp - - - - - try { - Thread.sleep(Constant.update_dataInfo_sleep_time); - } catch (InterruptedException e) { - } - } - -} diff --git a/src/com/platform/utils/ThreadVolume.java b/src/com/platform/utils/ThreadVolume.java index b7d49086..635ca7a8 100644 --- a/src/com/platform/utils/ThreadVolume.java +++ b/src/com/platform/utils/ThreadVolume.java @@ -2,9 +2,14 @@ package com.platform.utils; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import org.springframework.stereotype.Service; + +import com.platform.entities.Brick; import com.platform.entities.FolderNode; import com.platform.entities.VolumeEntity; +import com.platform.glusterfs.ClusterInfo; import com.platform.glusterfs.GetTreeData; import com.platform.glusterfs.VolumeInfo; @@ -18,6 +23,8 @@ public class ThreadVolume extends Thread implements Runnable{ /** Volume信息查询 */ private VolumeInfo volumeInfo = new VolumeInfo(); + private ClusterInfo cluster = new ClusterInfo(); + public ThreadVolume() { // TODO Auto-generated constructor stub } @@ -35,26 +42,70 @@ public class ThreadVolume extends Thread implements Runnable{ public void run() { super.run(); while(true){ - List folderlist = new ArrayList(); - //查询 volume name - List volumeNameList = volumeInfo.showAllVolumeName(); - if (null != volumeNameList) { - for (String volumeName : volumeNameList) { - VolumeEntity volume = new VolumeEntity(); - volume.setName(volumeName); - List path = volumeInfo.getVolumeMountPoint(volumeName); - //默认加载第一个路径 - if (null != path && path.size() > 0) { - //装入 folder： - //查询 每个 volume 下的 folder - FolderNode foldertmp = gfsTree.getDatas(path.get(0)); - folderlist.add(foldertmp); + try { + List folderlist = new ArrayList(); + List volumeList = new ArrayList(); + // brick状态 map集合 + Map brickStatusMap = cluster.showClusterInfo(); + + //查询 volume name + List volumeNameList = volumeInfo.showAllVolumeName(); + if (null != volumeNameList) { + for (String volumeName : volumeNameList) { + VolumeEntity volume = new VolumeEntity(); + volume.setName(volumeName); + List path = volumeInfo.getVolumeMountPoint(volumeName); + //默认加载第一个路径 + if (null != path && path.size() > 0) { + volume.setPath(path.get(0)); + } + volume.setAllSize(volumeInfo.getVolumeAllSize(volumeName)); + volume.setUsedSize(volumeInfo.getVolumeUseSize(volumeName)); + //TODO 查询brick-- + //返回 ip:path + List brickStrs = volumeInfo.getVolumeBricks(volumeName); + //brick已用大小： + Map usedSize = volumeInfo.getVolumebricksDataSize(volumeName); + Map availableSize = volumeInfo.getVolumebricksAvailableSize(volumeName); + + List brickList = new ArrayList(); + for (String brickIpPath : brickStrs) { + Brick b = new Brick(); + String ipAndpath[] = brickIpPath.split(":"); + String brickip = ipAndpath[0]; + String brickpath = ipAndpath[1]; + //iP , path , + b.setIp(brickip); + if(brickStatusMap==null || brickStatusMap.size()==0){ + b.setStatus(false); + }else if (brickStatusMap.containsKey(brickip)) { + b.setStatus(true); + } + else { + b.setStatus(false); + } + b.setPath(brickpath); + b.setAvailableSize(availableSize.get(brickIpPath)); + b.setUsedSize(usedSize.get(brickIpPath)); + brickList.add(b); + } + volume.setBrick(brickList); + + + //默认加载第一个路径 + if (null != path && path.size() > 0) { + //装入 folder： + //查询 每个 volume 下的 folder + FolderNode foldertmp = gfsTree.getDatas(path.get(0)); + folderlist.add(foldertmp); + } + volumeList.add(volume); } } - } - //TODO 更新folder 目录 - CacheTreeData.setFolders(folderlist); - try { + //TODO 更新folder 目录 + CacheTreeData.setFolders(folderlist); + CacheTreeData.setVolumeList(volumeList); + Thread.sleep(Constant.get_volume_sleep_time); } catch (InterruptedException e) { }