diff --git a/src/com/dao/mapper/data-details-mapper.xml b/src/com/dao/mapper/data-details-mapper.xml index 89b9f52e..5cd57968 100644 --- a/src/com/dao/mapper/data-details-mapper.xml +++ b/src/com/dao/mapper/data-details-mapper.xml @@ -113,7 +113,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" regionalism_code, - + system_code, diff --git a/src/com/dao/mapper/data-detaisl-mapper.xml b/src/com/dao/mapper/data-detaisl-mapper.xml deleted file mode 100644 index d0f8d7c7..00000000 --- a/src/com/dao/mapper/data-detaisl-mapper.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - and data_details.data_type=#{PagerOptions.dataType} - - - and - data_details.submitted_batch=#{PagerOptions.submittedBatch} - - - and data_details.city_name=#{PagerOptions.cityName} - - - and - data_details.district_name=#{PagerOptions.districtName} - - - and data_details.data_version=#{PagerOptions.dataVersion} - - - and data_details=#{PagerOptions.systemName} - - - and data_details.data_year=#{PagerOptions.dataYear} - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/com/dao/mapper/dataInfoMoveTmpmapper.xml b/src/com/dao/mapper/dataInfoMoveTmpmapper.xml index 7d1145d6..bf4987f4 100644 --- a/src/com/dao/mapper/dataInfoMoveTmpmapper.xml +++ b/src/com/dao/mapper/dataInfoMoveTmpmapper.xml @@ -43,10 +43,12 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" jdbcType="INTEGER" /> + + - regionalism_code,system_code,dst_path,lasttime + regionalism_code,system_code,dst_path,lasttime,fkid @@ -54,12 +56,12 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" SELECT a.id id,a.regionalism_code regionalismCode,b.city_name cityName,b.district_name districtName, a.system_code systemCode,b.system_name systemName,b.data_type dataType,b.data_version dataVersion, - b.submitted_batch submittedBatch,b.data_path dataPath,b.data_charset charset,b.collection_time collectionTime, + b.submitted_batch submittedBatch,b.data_path dataPath,b.data_charset charset,b.collection_time collectingTime, b.collector_name collectorName,b.collector_contacts collectorContacts,b.data_year dataYear,a.dst_path dstPath, a.complete_status completeStatus,a.rate rate, a.lasttime lastTime FROM move_data_tmp a LEFT JOIN data_details b - ON a.system_code = b.system_code AND a.regionalism_code = b.regionalism_code + ON a.system_code = b.system_code AND a.regionalism_code = b.regionalism_code AND a.fkid = b.id ORDER BY a.id @@ -112,6 +114,9 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" lasttime, + + fkid, + ) VALUES( @@ -137,6 +142,9 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" #{lastTime}, + + #{fkid), + ) @@ -145,7 +153,7 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN" INSERT INTO move_data_tmp ( ) VALUES - (#{item.regionalismCode,jdbcType=VARCHAR},#{item.systemCode,jdbcType=INTEGER},#{item.dstPath,jdbcType=VARCHAR},#{item.lastTime,jdbcType=VARCHAR}) + (#{item.regionalismCode,jdbcType=VARCHAR},#{item.systemCode,jdbcType=INTEGER},#{item.dstPath,jdbcType=VARCHAR},#{item.lastTime,jdbcType=VARCHAR},#{item.fkid}) diff --git a/src/com/platform/controller/DataModelController.java b/src/com/platform/controller/DataModelController.java index 7cf47830..743b3bd3 100644 --- a/src/com/platform/controller/DataModelController.java +++ b/src/com/platform/controller/DataModelController.java @@ -16,9 +16,11 @@ import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import org.springframework.stereotype.Controller; import org.springframework.stereotype.Service; import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; @@ -31,6 +33,7 @@ import com.platform.entities.GatherOracleInfo; import com.platform.entities.OracleConnectorParams; import com.platform.entities.PagerOptions; import com.platform.entities.VolumeEntity; +import com.platform.glusterfs.CheckoutMD5; import com.platform.service.DataInfoService; import com.platform.service.IGfsService; import com.platform.service.IMoveDataService; @@ -48,6 +51,9 @@ import com.platform.utils.UtilsHelper; @Controller public class DataModelController extends BaseController{ + + public static Logger log = Logger.getLogger(DataModelController.class); + @Resource(name = "dataInfoService") private DataInfoService dfs; @@ -62,7 +68,7 @@ public class DataModelController extends BaseController{ @Resource(name = "moveDataService") private IMoveDataService moveDataService; - + public void setDfsImp(DataInfoService dfs) { this.dfs = dfs; } @@ -129,7 +135,7 @@ public class DataModelController extends BaseController{ } @RequestMapping(value="/oracle/{name}/extract", method= RequestMethod.POST) - public void extractOracleData(HttpServletRequest res, + public void oracleExtract(HttpServletRequest res, HttpServletResponse req, String name) throws Exception { res.setCharacterEncoding("UTF-8"); System.out.println("------extract-------"); @@ -155,23 +161,44 @@ public class DataModelController extends BaseController{ OracleExtract.extractOracle(name, colleclist, map); } +// public void oracleExtract(HttpServletRequest res, +// HttpServletResponse req, String name, @RequestBody GatherOracleInfo gather, @RequestBody List collectOracles) +// throws Exception { +//// res.setCharacterEncoding("UTF-8"); +// log.debug("------extract-------"); +//// Map paramMap = res.getParameterMap(); +//// log.debug(paramMap); +//// //汇总库 对象信息--带有tableName- +//// String[] nodes = paramMap.get("target"); +//// Map map = null; +//// for (String string : nodes) { +//// JSONObject jsonobject = JSONObject.fromObject(string); +//// map = jsonobject; +//// System.out.println(map); +//// } +//// +//// // 采集库对象--(多个采集库抽取到1个汇总库的1个tableName下) +//// String[] inneed = paramMap.get("inneed"); +//// List> colleclist = new ArrayList>(); +//// for (String string : nodes) { +//// JSONObject jsonobject = JSONObject.fromObject(string); +//// Map inneedMap = jsonobject; +//// colleclist.add(inneedMap); +//// } +// OracleExtract.extractOracle(name, collectOracles, gather); +// } @RequestMapping(value="/volume/list", method= RequestMethod.POST) @ResponseBody - public List getFolder(HttpServletRequest res, HttpServletResponse req) throws Exception { - System.out.println("get Request"); -// Brick brick1 = new Brick("192.168.0.101", "D:/bootstrap"); -// Brick brick2 = new Brick("192.168.0.103", "D:\book"); -// List bricks = new ArrayList(); -// bricks.add(brick1); -// bricks.add(brick2); + public List volumeList() throws Exception { + log.debug("-----/volume/list------"); List result = gfsService.getAllVolumes(); return result; } @RequestMapping(value = "/volume/{name}/move", method= RequestMethod.POST) - @ResponseBody - public Object move(HttpServletRequest res, HttpServletResponse req, String name, +// @ResponseBody + public void volumeMove(HttpServletRequest res, HttpServletResponse req, String name, FolderNode selectNode, DataInfoEntity selectItems) throws Exception { res.setCharacterEncoding("UTF-8"); System.out.println("get Request"); @@ -195,71 +222,88 @@ public class DataModelController extends BaseController{ System.out.println(string); JSONObject jsobj = JSONObject.fromObject(string); Map itemmap = jsobj; - if (null != itemmap.get("volume")) { - srcVolumes.add((String) itemmap.get("volume")); - } DataInfoEntity data = (DataInfoEntity) Bean2MapUtils.convertMap(DataInfoEntity.class, itemmap); datas.add(data); } System.out.println("------/volume/{name}/move--------"); - boolean result = false ; if (datas.size() >0) { - result = moveDataService.moveData(datas, (String)map.get("path")); + moveDataService.moveData(datas, (String)map.get("path")); } - return result; } +// public Object volumeMove(HttpServletRequest res, HttpServletResponse req, String name, +// @RequestBody FolderNode selectNode, @RequestBody List selectItems) throws Exception { +// log.debug("-----/volume/{name}/move------"); +//// Map paramMap = res.getParameterMap(); +//// System.out.println(paramMap); +// Map paramMap = res.getParameterMap(); +// log.debug(paramMap); +// //汇总库 对象信息--带有tableName- +// String[] nodes = (String[]) paramMap.get("selectNode"); +// Map map = null; +// JSONObject jsonobject = JSONObject.fromObject(string); +// map = jsonobject; +// +// +// log.debug(selectItems); +// log.debug(selectNode); +// boolean result = false ; +// if (selectItems.size() >0) { +// result = moveDataService.moveData(selectItems, selectNode.getPath()); +// } +// return result; +// } @RequestMapping(value="/oracle/list", method= RequestMethod.POST) @ResponseBody - public List getOracleInfo(HttpServletRequest res, - HttpServletResponse req) throws Exception { - + public List oracleList() throws Exception { + log.debug("----------getOracleInfo-----------------------"); List result = mySqlService.findAllMySql(); - System.out.println("----------getOracleInfo-----------------------"); return result; } @RequestMapping(value="/oracle/{id}/delete", method= RequestMethod.POST) @ResponseBody - public void deleteOracleInfo(HttpServletRequest res, - HttpServletResponse req, int id) throws Exception { - res.setCharacterEncoding("UTF-8"); + public void oracleDelete(int id) throws Exception { + log.debug("----------deleteOracleInfo----------------"); mySqlService.deleteMySql(id); - System.out.println("----------deleteOracleInfo-----------------------"); } @RequestMapping(value="/oracle/{id}/insert", method= RequestMethod.POST) @ResponseBody - public void insertOracleInfo(HttpServletRequest res, - HttpServletResponse req, GatherOracleInfo oracle) throws Exception { - res.setCharacterEncoding("UTF-8"); + public void oracleInsert(GatherOracleInfo oracle) throws Exception { + log.debug("----------insertOracleInfo----------------"); mySqlService.insertOracle(oracle); - System.out.println("----------insertOracleInfo-----------------------"); } @RequestMapping(value="/oracle/{id}/update", method= RequestMethod.POST) - @ResponseBody - public void updateOracleInfo(HttpServletRequest res, - HttpServletResponse req, GatherOracleInfo oracle) throws Exception { - res.setCharacterEncoding("gb2312"); - System.out.println(oracle.getName()); - System.out.println(oracle); - + public void oracleUpdate(@RequestBody GatherOracleInfo oracle) throws Exception { + log.debug("----------updateOracleInfo-----------------------"); + log.debug(oracle); mySqlService.updateOracle(oracle); - System.out.println("----------updateOracleInfo-----------------------"); } @RequestMapping(value="/task/transfer/list", method= RequestMethod.POST) @ResponseBody - public Object taskList() throws Exception { + public Object taskTransferList() throws Exception { + log.debug("----------/task/transfer/list-----------------------"); List result = moveDataService.findAll(); return result; } @RequestMapping(value="/task/transfer/delete", method= RequestMethod.POST) @ResponseBody - public Object taskdeletes(DataInfoEntityMoveTmp move) throws Exception { + public Object taskTransferDelete(DataInfoEntityMoveTmp move) throws Exception { + log.debug("----------/task/transfer/delete-----------------------"); int result = moveDataService.delete(move); return result; } + + @RequestMapping(value="/task/transfer/save", method= RequestMethod.POST) + @ResponseBody + public Object taskTransferSave(DataInfoEntity move) throws Exception { + log.debug("---------/task/transfer/save-----------------------"); + int result = dfs.save(move); + return result; + } + } diff --git a/src/com/platform/entities/DataInfoEntityMoveTmp.java b/src/com/platform/entities/DataInfoEntityMoveTmp.java index 0fb469e5..1b1b6e10 100644 --- a/src/com/platform/entities/DataInfoEntityMoveTmp.java +++ b/src/com/platform/entities/DataInfoEntityMoveTmp.java @@ -10,6 +10,8 @@ public class DataInfoEntityMoveTmp extends DataInfoEntity { private String lastTime; // 采集时间 + private int fkid; + public DataInfoEntityMoveTmp() { // TODO Auto-generated constructor stub } @@ -70,4 +72,18 @@ public class DataInfoEntityMoveTmp extends DataInfoEntity { this.lastTime = lastTime; } + /** + * @return the fkid + */ + public int getFkid() { + return fkid; + } + + /** + * @param fkid the fkid to set + */ + public void setFkid(int fkid) { + this.fkid = fkid; + } + } diff --git a/src/com/platform/glusterfs/CopyData.java b/src/com/platform/glusterfs/CopyData.java index 86289174..a8cbdde3 100644 --- a/src/com/platform/glusterfs/CopyData.java +++ b/src/com/platform/glusterfs/CopyData.java @@ -36,6 +36,38 @@ public class CopyData { status = copyFolderFilesAnyway(sourceFolderName, destFolderName, fileName); return status; } + + /** + * 将sourceFolderName拷贝到destFolderName + * 如果拷贝正常返回1，如果sourceFolderName不存在返回-2 ，如果destFolderName不存在返回-3 + * @param sourceFolderName + * @param destFolderName + * @return + * @see [类、类#方法、类#成员] + */ + public int copyFolder(String sourceFolderName, String destFolderName) { + createFolders(destFolderName); + int progress=0; + log.info("start copy " + sourceFolderName + " to " + destFolderName); + ShowData showData=new ShowData(); + Map reStrings=showData.showFolderData(destFolderName); + if(reStrings==null){ + log.info("3201 "+destFolderName+" is not exists"); + return -3; + } + + reStrings=showData.showFolderData(sourceFolderName); + if(reStrings==null){ + log.info("3202 "+sourceFolderName+" is not exists"); + return -2; + } + String command = "cp -r " + sourceFolderName+" "+destFolderName; + + Constant.ganymedSSH.execCmdNoWaitAcquiescent(command); + + log.info("copy " + sourceFolderName + " to " + destFolderName + " running"); + return 1; + } /** * -1 :error; -2: the filename is not exists ;-3 :destFolderName ; 1: right diff --git a/src/com/platform/service/DataInfoServiceImp.java b/src/com/platform/service/DataInfoServiceImp.java index 4da74444..47aeb118 100644 --- a/src/com/platform/service/DataInfoServiceImp.java +++ b/src/com/platform/service/DataInfoServiceImp.java @@ -65,7 +65,13 @@ public class DataInfoServiceImp implements DataInfoService { @Override public int save(DataInfoEntity data) throws Exception { - int result = dfdDao.save(data); + int result = 0; + try { + result = dfdDao.save(data); + } catch (Exception e) { + // TODO: handle exception + System.out.println(""); + } return result; } } diff --git a/src/com/platform/service/IOracleExtractService.java b/src/com/platform/service/IOracleExtractService.java index be02d6d9..7d4e9418 100644 --- a/src/com/platform/service/IOracleExtractService.java +++ b/src/com/platform/service/IOracleExtractService.java @@ -3,6 +3,10 @@ package com.platform.service; import java.util.List; import java.util.Map; +import com.platform.entities.DataInfoEntity; +import com.platform.entities.GatherOracleInfo; +import com.platform.entities.OracleConnectorParams; + public interface IOracleExtractService { /** 抽取数据库 @@ -12,6 +16,7 @@ public interface IOracleExtractService { * @return * @throws Exception */ +// public boolean extractOracle(String name, List dataInfos, GatherOracleInfo oracleConnect) throws Exception; +// public boolean extractOracle(String name, List dataInfos, Map oracleConnect) throws Exception; public boolean extractOracle(String name, List> dataInfo, Map oracleConnect) throws Exception; - } diff --git a/src/com/platform/service/impl/MoveDataServiceImpl.java b/src/com/platform/service/impl/MoveDataServiceImpl.java index 511676a7..007fd0ee 100644 --- a/src/com/platform/service/impl/MoveDataServiceImpl.java +++ b/src/com/platform/service/impl/MoveDataServiceImpl.java @@ -18,6 +18,7 @@ import com.platform.entities.DataInfoEntity; import com.platform.entities.DataInfoEntityMoveTmp; import com.platform.glusterfs.CheckoutMD5; import com.platform.glusterfs.CopyData; +import com.platform.glusterfs.RemoveData; import com.platform.glusterfs.ShowData; import com.platform.service.IMoveDataService; import com.platform.utils.Bean2MapUtils; @@ -31,6 +32,8 @@ public class MoveDataServiceImpl implements IMoveDataService{ @Resource(name = "dataInfoDao") private DataInfoDao dataInfoDao; + private RemoveData removeservice = new RemoveData(); + /** * 迁移 */ @@ -91,6 +94,7 @@ public class MoveDataServiceImpl implements IMoveDataService{ dataMove.setRegionalismCode(dataInfoEntity.getRegionalismCode()); dataMove.setDstPath(finalDestPath); dataMove.setLastTime(DateForm.date2StringBysecond(new Date())); + dataMove.setFkid(dataInfoEntity.getId()); moveList.add(dataMove); } if (moveList.size() > 0) { @@ -115,6 +119,9 @@ public class MoveDataServiceImpl implements IMoveDataService{ @Override public int delete(DataInfoEntityMoveTmp dataMove) throws Exception { + if ("1".equals(dataMove.getCompleteStatus())) { + removeservice.deleteFolder(dataMove.getDstPath()); + } int result = dataInfoMoveTmpDao.remove(dataMove.getId()); return result; } diff --git a/src/com/platform/service/impl/OracleExtractServiceImpl.java b/src/com/platform/service/impl/OracleExtractServiceImpl.java index 9d2a36e5..9cf58a00 100644 --- a/src/com/platform/service/impl/OracleExtractServiceImpl.java +++ b/src/com/platform/service/impl/OracleExtractServiceImpl.java @@ -27,7 +27,7 @@ public class OracleExtractServiceImpl implements IOracleExtractService { * 数据库连接实现类 */ private OracleConnector connect = new OracleConnector(); - + @Override public boolean extractOracle(String name, List> dataInfoMap, Map oracleConnect) throws Exception { @@ -60,4 +60,35 @@ public class OracleExtractServiceImpl implements IOracleExtractService { return false; } +// @Override +// public boolean extractOracle(String name, List dataInfos, GatherOracleInfo oracleConnect) throws Exception { +// boolean isSuccess = false; +// try{ +// //map转 bean(汇总库信息-带tableName的) +//// GatherOracleInfo oracleModel = (GatherOracleInfo) Bean2MapUtils.convertMap(GatherOracleInfo.class, oracleConnect); +// +// //采集库连接参数 +//// List datainfos = new ArrayList(); +//// for (Map map : dataInfoMap) { +//// OracleConnectorParams dataInfoEntity = (OracleConnectorParams) Bean2MapUtils.convertMap(OracleConnectorParams.class, oracleConnect); +//// datainfos.add(dataInfoEntity); +//// } +// +// Connection conn = OracleConnector.ConnectionBuilder("jdbc:oracle:thin:@" + oracleConnect.getIp() + ":" + oracleConnect.getPort() + "/" +// + oracleConnect.getDatabaseName(), oracleConnect.getUser(), oracleConnect.getPassword()); +// +// for (OracleConnectorParams collectOracle : dataInfos) { +// +// oracleExtract.createDBLink(conn, collectOracle); +// oracleExtract.createTableSpace(conn, oracleConnect); +// oracleExtract.createUser(conn, oracleConnect); +// oracleExtract.extractColleDB(conn, collectOracle); +// } +// isSuccess = true; +// }catch(Exception e){ +// +// } +// return false; +// } + } diff --git a/src/com/platform/service/thread/ThreadMoveData.java b/src/com/platform/service/thread/ThreadMoveData.java index b7e2f867..dc6b60ec 100644 --- a/src/com/platform/service/thread/ThreadMoveData.java +++ b/src/com/platform/service/thread/ThreadMoveData.java @@ -4,9 +4,12 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Resource; +import org.apache.log4j.Logger; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; @@ -31,6 +34,8 @@ import com.platform.utils.DateForm; @Component public class ThreadMoveData{ + public static Logger log = Logger.getLogger(ThreadMoveData.class); + @Resource(name = "dataInfoDao") private DataInfoDao dataInfoDao; @@ -88,12 +93,22 @@ public class ThreadMoveData{ // 查询大小：。//gfs 获取size， long srcSize = show.getFolderSize(dataMove.getDataPath()); long dstSize = show.getFolderSize(dataMove.getDstPath()); + if (srcSize < 0) { + log.error(dataMove.getDataPath() + " : 路径不存在！ "); + continue; + } + if (dstSize < 0) { + log.error(dataMove.getDstPath() + " : 路径不存在！ "); + continue; + } if (srcSize > 0 && dstSize > 0) { - realRate = dstSize / srcSize * 100; + realRate = (dstSize*100 / srcSize ); dataMove.setRate((int) realRate); dataMove.setLastTime(DateForm.date2StringBysecond(new Date())); } - + if (srcSize == dstSize) { + realRate = 100; + } } if("1".equals(dataMove.getCompleteStatus()) &&dataMove.getRate() > 0){ //传输完毕：进行校验 @@ -110,7 +125,7 @@ public class ThreadMoveData{ DataInfoEntity data = (DataInfoEntity) Bean2MapUtils.convertMap( DataInfoEntity.class, Bean2MapUtils.convertBean(dataMove)); data.setDataPath(dataMove.getDstPath()); - data.setYear(dataMove.getYear()+" (迁移完成时间 " + DateForm.date2StringByMin(new Date())+")"); + data.setYear(DateForm.date2StringByMin(new Date())); dataInfoDao.save(data); } else { @@ -135,7 +150,15 @@ public class ThreadMoveData{ //如果是 待 迁移状态的 if ("0".equals(next2move.getCompleteStatus())) { //待迁移的数据 -- 开始迁移 - copy.copyFolderFilesAnyway(next2move.getDataPath(), next2move.getDstPath(), "app"); + // 末尾 含有 / + Pattern pattern2 = Pattern.compile("\\/$"); + String dstPath = next2move.getDataPath(); + Matcher matcher2 = pattern2.matcher(dstPath); + //去掉 最后 的 / 符合 + if (matcher2.find()) { + dstPath = dstPath.substring(0, dstPath.length()-1); + } + copy.copyFolder(dstPath+"/app", next2move.getDstPath()); // "1" :正在上传，0：等待 迁移， 2：成功 3：失败 next2move.setCompleteStatus("1"); next2move.setLastTime(DateForm.date2StringBysecond(new Date()));