innov
/
aggregation-platform
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

完善抽取功能

Browse Source
web_backend_develope
wu ming 9 years ago
parent 34e31fbc6d ee7128c617
commit f645d6e119
5 changed files with 170 additions and 154 deletions
Show Stats Download Patch File Download Diff File

184
src/com/platform/controller/DataModelController.java
Unescape View File

@ -42,31 +42,31 @@ import com.platform.utils.Configs;
import com.platform.utils.UtilsHelper;
@Controller
public class DataModelController extends BaseController{
public class DataModelController extends BaseController {
public static Logger log = Configs.DAILY_ROLLING_LOGGER;
@Resource(name = "dataInfoService")
private DataInfoService dfs;
@Resource(name = "gfsService")
private IGfsService gfsService;
@Resource(name = "mySqlService")
private IMySqlService mySqlService;
@Resource(name = "OracleExtract")
private IOracleExtractService OracleExtract;
@Resource(name = "moveDataService")
private IMoveDataService moveDataService;
@Resource(name = "codeService")
private ICodeService codeService;
@Resource(name = "logReadService")
private ILogRead logReadService;
public void setDfsImp(DataInfoService dfs) {
this.dfs = dfs;
}
@ -93,12 +93,13 @@ public class DataModelController extends BaseController{
.append("}").toString());
PagerOptions pagerOptions = (PagerOptions) UtilsHelper
.newObjAndSetAttrsByClass(PagerOptions.class, params);
return dfs.getPagerTableData(pagerOptions);
}
@RequestMapping(value="/delete/data", method= RequestMethod.POST)
public void deleteData(HttpServletRequest res, HttpServletResponse req) throws Exception {
@RequestMapping(value = "/delete/data", method = RequestMethod.POST)
public void deleteData(HttpServletRequest res, HttpServletResponse req)
throws Exception {
res.setCharacterEncoding("UTF-8");
Map<String, String[]> paramMap = res.getParameterMap();
String[] data = paramMap.get("data");
@ -107,7 +108,8 @@ public class DataModelController extends BaseController{
}
@RequestMapping("/connectOracle")
public void connectOracle(HttpServletRequest res, HttpServletResponse req) throws UnsupportedEncodingException {
public void connectOracle(HttpServletRequest res, HttpServletResponse req)
throws UnsupportedEncodingException {
res.setCharacterEncoding("UTF-8");
Map<String, String[]> paramMap = res.getParameterMap();
String[] oraclesName = paramMap.get("oracleName");
@ -135,54 +137,59 @@ public class DataModelController extends BaseController{
req.setStatus(200);
}
@RequestMapping(value="/oracle/{name}/extract", method= RequestMethod.POST)
public void oracleExtract(HttpServletRequest res,
HttpServletResponse req,@RequestBody oracleForm form) throws Exception {
@RequestMapping(value = "/oracle/{name}/extract", method = RequestMethod.POST)
public void oracleExtract(HttpServletRequest res, HttpServletResponse req,
@RequestBody oracleForm form) throws Exception {
Configs.CONSOLE_LOGGER.error("/oracle/{name}/extract");
res.setCharacterEncoding("UTF-8");
if (null != form.getInneed() && form.getInneed().size() > 0) {
OracleExtract.extractOracle(form.getName(), form.getInneed(), form.getTarget());
OracleExtract.extractOracle(form.getName(), form.getInneed(),
form.getTarget());
}
req.setStatus(200);
}
// public void oracleExtract(HttpServletRequest res,
// HttpServletResponse req, String name, @RequestBody GatherOracleInfo gather, @RequestBody List<OracleConnectorParams> collectOracles)
// throws Exception {
//// res.setCharacterEncoding("UTF-8");
// log.debug("------extract-------");
//// Map<String, String[]> paramMap = res.getParameterMap();
//// log.debug(paramMap);
//// //汇总库 对象信息--带有tableName-
//// String[] nodes = paramMap.get("target");
//// Map map = null;
//// for (String string : nodes) {
//// JSONObject jsonobject = JSONObject.fromObject(string);
//// map = jsonobject;
//// System.out.println(map);
//// }
////
//// // 采集库对象--(多个采集库抽取到1个汇总库的1个tableName下)
//// String[] inneed = paramMap.get("inneed");
//// List<Map<String, String>> colleclist = new ArrayList<Map<String,String>>();
//// for (String string : nodes) {
//// JSONObject jsonobject = JSONObject.fromObject(string);
//// Map inneedMap = jsonobject;
//// colleclist.add(inneedMap);
//// }
// OracleExtract.extractOracle(name, collectOracles, gather);
// }
@RequestMapping(value="/volume/list", method= RequestMethod.POST)
// public void oracleExtract(HttpServletRequest res,
// HttpServletResponse req, String name, @RequestBody GatherOracleInfo
// gather, @RequestBody List<OracleConnectorParams> collectOracles)
// throws Exception {
// // res.setCharacterEncoding("UTF-8");
// log.debug("------extract-------");
// // Map<String, String[]> paramMap = res.getParameterMap();
// // log.debug(paramMap);
// // //汇总库 对象信息--带有tableName-
// // String[] nodes = paramMap.get("target");
// // Map map = null;
// // for (String string : nodes) {
// // JSONObject jsonobject = JSONObject.fromObject(string);
// // map = jsonobject;
// // System.out.println(map);
// // }
// //
// // // 采集库对象--(多个采集库抽取到1个汇总库的1个tableName下)
// // String[] inneed = paramMap.get("inneed");
// // List<Map<String, String>> colleclist = new
// ArrayList<Map<String,String>>();
// // for (String string : nodes) {
// // JSONObject jsonobject = JSONObject.fromObject(string);
// // Map inneedMap = jsonobject;
// // colleclist.add(inneedMap);
// // }
// OracleExtract.extractOracle(name, collectOracles, gather);
// }
@RequestMapping(value = "/volume/list", method = RequestMethod.POST)
@ResponseBody
public List<VolumeEntity> volumeList() throws Exception {
log.debug("-----/volume/list------");
List<VolumeEntity> result = gfsService.getAllVolumes();
return result;
}
@RequestMapping(value = "/volume/{name}/move", method= RequestMethod.POST)
@RequestMapping(value = "/volume/{name}/move", method = RequestMethod.POST)
@ResponseBody
public void volumeMove(HttpServletRequest res, HttpServletResponse req,@RequestBody volumeMoveForm form) throws Exception {
public void volumeMove(HttpServletRequest res, HttpServletResponse req,
@RequestBody volumeMoveForm form) throws Exception {
List<DataInfoEntity> datas = new ArrayList<DataInfoEntity>();
List<DataInfoEntity> selectItems = form.getSelectItems();
if (null == selectItems) {
@ -190,94 +197,103 @@ public class DataModelController extends BaseController{
return;
}
datas.addAll(selectItems);
System.out.println("------/volume/{name}/move--------");
if (datas.size() >0) {
if (datas.size() > 0) {
moveDataService.moveData(datas, form.getSelectNode());
}
req.setStatus(200);
}
@RequestMapping(value="/oracle/list", method= RequestMethod.POST)
@RequestMapping(value = "/oracle/list", method = RequestMethod.POST)
@ResponseBody
public List<GatherOracleInfo> oracleList() throws Exception {
log.debug("----------getOracleInfo-----------------------");
List<GatherOracleInfo> result = mySqlService.findAllMySql();
List<GatherOracleInfo> result = mySqlService.findAllMySql();
return result;
}
@RequestMapping(value="/oracle/{id}/delete", method= RequestMethod.POST)
public void oracleDelete(HttpServletRequest req, HttpServletResponse res, @PathVariable String id) throws Exception {
@RequestMapping(value = "/oracle/{id}/delete", method = RequestMethod.POST)
public void oracleDelete(HttpServletRequest req, HttpServletResponse res,
@PathVariable String id) throws Exception {
log.debug("----------deleteOracleInfo----------------");
Integer num = Integer.valueOf(id);
Integer result = mySqlService.deleteMySql(num);
Integer result = mySqlService.deleteMySql(num);
log.info("oracleDelete : " + result);
res.setStatus(200);
}
@RequestMapping(value="/oracle/{id}/insert", method= RequestMethod.POST)
@RequestMapping(value = "/oracle/{id}/insert", method = RequestMethod.POST)
@ResponseBody
public void oracleInsert(HttpServletRequest res, HttpServletResponse req, @PathVariable String id, @RequestBody GatherOracleInfo oracle) throws Exception {
public void oracleInsert(HttpServletRequest res, HttpServletResponse req,
@PathVariable String id, @RequestBody GatherOracleInfo oracle)
throws Exception {
log.debug("----------insertOracleInfo----------------");
oracle.setId(Integer.valueOf(id));
mySqlService.insertOracle(oracle);
req.setStatus(200);
}
@RequestMapping(value="/oracle/{id}/update", method= RequestMethod.POST)
@RequestMapping(value = "/oracle/{id}/update", method = RequestMethod.POST)
@ResponseBody
public void oracleUpdate(HttpServletRequest res, HttpServletResponse req, @PathVariable("id") String id, @RequestBody GatherOracleInfo oracle) throws Exception {
public void oracleUpdate(HttpServletRequest res, HttpServletResponse req,
@PathVariable("id") String id, @RequestBody GatherOracleInfo oracle)
throws Exception {
log.debug("----------updateOracleInfo-----------------------");
log.debug(oracle);
mySqlService.updateOracle(oracle);
mySqlService.updateOracle(oracle);
req.setStatus(200);
}
@RequestMapping(value="/task/transfer/list", method= RequestMethod.POST)
@RequestMapping(value = "/task/transfer/list", method = RequestMethod.POST)
@ResponseBody
public Object taskTransferList() throws Exception {
log.debug("----------/task/transfer/list-----------------------");
List<DataInfoEntityMoveTmp> result = moveDataService.findAll();
return result;
}
@RequestMapping(value="/task/transfer/{id}/delete", method= RequestMethod.POST)
@RequestMapping(value = "/task/transfer/{id}/delete", method = RequestMethod.POST)
@ResponseBody
public Object taskTransferDelete(HttpServletRequest res, HttpServletResponse req, @PathVariable String id) throws Exception {
public Object taskTransferDelete(HttpServletRequest res,
HttpServletResponse req, @PathVariable String id) throws Exception {
log.debug("----------/task/transfer/{id}/delete-----------------------");
DataInfoEntityMoveTmp move = new DataInfoEntityMoveTmp();
move.setId(Integer.valueOf(id));
int result = moveDataService.delete(move );
int result = moveDataService.delete(move);
req.setStatus(200);
return result;
}
@RequestMapping(value="/task/transfer/save", method= RequestMethod.POST)
@RequestMapping(value = "/task/transfer/save", method = RequestMethod.POST)
@ResponseBody
public Object taskTransferSave(HttpServletRequest res, HttpServletResponse req, @RequestBody DataInfoEntity move) throws Exception {
public Object taskTransferSave(HttpServletRequest res,
HttpServletResponse req, @RequestBody DataInfoEntity move)
throws Exception {
log.debug("---------/task/transfer/save-----------------------");
int result = dfs.save(move);
req.setStatus(200);
return result;
}
@RequestMapping(value="/code/list", method= RequestMethod.POST)
@RequestMapping(value = "/code/list", method = RequestMethod.POST)
@ResponseBody
public Object findCodeList() throws Exception {
log.debug("---------/findSystemCode-----------------------");
Map<String, Object> result = codeService.findAll();
return result;
}
@RequestMapping(value = "/oracle/extract/log", method= RequestMethod.POST )
@RequestMapping(value = "/oracle/extract/log", method = RequestMethod.POST)
@ResponseBody
public Object getExtractLog(@RequestParam("rcName") String name, HttpServletRequest res, HttpServletResponse req) throws Exception{
public Object getExtractLog(@RequestParam("rcName") String name,
HttpServletRequest res, HttpServletResponse req) throws Exception {
log.info("---------/oracle/extract/log-------------------");
String result = logReadService.readLog(name);
// StringBuilder sb = new StringBuilder();
// sb.append("查看相应日志").append("\n").append("看到了");
// Map<String, String> log = new HashMap<>();
// log.put(name, sb.toString());
return result;
// StringBuilder sb = new StringBuilder();
// sb.append("查看相应日志").append("\n").append("看到了");
Map<String, String> log = new HashMap<>();
log.put(name, result);
return log;
}
}

85
src/com/platform/service/OracleExtractHelper.java
Unescape View File

@ -22,7 +22,7 @@ public class OracleExtractHelper {
* dblink的名称
* @return
*/
private boolean hasSameNameDBLink(Connection conn, String linkName) {
private boolean hasSameNameDBLink(Connection conn, String linkName, String filePath) {
boolean flag = false;
String sql = "SELECT * FROM ALL_DB_LINKS WHERE DB_LINK='" + linkName+"'";
ResultSet rSet = null;
@ -31,9 +31,11 @@ public class OracleExtractHelper {
rSet.last();
if (rSet.getRow() > 0)
flag = true;
FileOperateHelper
.fileWrite(filePath, sql+ "\r\n"+"OK \r\n");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
FileOperateHelper
.fileWrite(filePath, sql+ "\r\n"+ e.getMessage() + " \r\n");
new CustomException(Custom4exception.threadVolume_Oracle_Except, e, rSet);
}
return flag;
@ -59,16 +61,16 @@ public class OracleExtractHelper {
+ ")))(CONNECT_DATA =(SERVICE_NAME =" + oc.getDatabaseName()
+ ")))\'";
if (null != oc) {
if (hasSameNameDBLink(conn, linkName)) { // 如果dblink已经存在,先删除dblink，在创建dblink
if (hasSameNameDBLink(conn, linkName, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) { // 如果dblink已经存在,先删除dblink，在创建dblink
String deleteSql = "DROP PUBLIC DATABASE LINK "
+ linkName;
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
deleteSql+"\r\n");
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// deleteSql+"\r\n");
if (OracleConnector.execOracleSQL(conn, deleteSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) {
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
+ oc.getName(), sql+"\r\n");
// FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
// + oc.getName(), sql+"\r\n");
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} else {
Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!");
@ -79,8 +81,8 @@ public class OracleExtractHelper {
} else {
// 否则，创建dblink
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
FileOperateHelper.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql);
// FileOperateHelper.fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(), sql);
}
}
}
@ -89,28 +91,29 @@ public class OracleExtractHelper {
* 创建表空间
*
* @param conn 汇总库连接
* @param collectOracle
* @param oc 汇总库信息
* @return
*/
public boolean createTableSpace(Connection conn, GatherOracleInfo oc) {
public boolean createTableSpace(Connection conn, OracleConnectorParams collectOracle, GatherOracleInfo oc) {
String tmpSql = "select TABLESPACE_NAME from dba_tablespaces where TABLESPACE_NAME = '"
+ oc.getTableName() + "'";
if (OracleConnector.execUpdateOracleSQL(conn, tmpSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) {
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
tmpSql+"\r\n");
if (OracleConnector.execUpdateOracleSQL(conn, tmpSql, Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName())) {
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// tmpSql+"\r\n");
return true;
} else {
String sql = "create tablespace " + oc.getTableName()
+ " datafile '" + Configs.GATHER_TABLESPACE_PATH
+ oc.getTableName() + ".dbf"
+ "' size 512M autoextend on next 512M maxsize unlimited";
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
sql+"\r\n");
return OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// sql+"\r\n");
return OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName());
}
}
@ -122,22 +125,23 @@ public class OracleExtractHelper {
*/
public void createUser(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) {
String strTUser = oc.getName() + totalOracle.getSuffix();
String sql = "Create user " + strTUser + " default tablespace "
+ totalOracle.getTableName() + " identified by "
+ Configs.GATHER_TABLE_PASSWORD;
String grantSql = "grant connect, resource, dba to " + strTUser;
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
sql+"\r\n");
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// sql+"\r\n");
OracleConnector.execOracleSQL(conn, grantSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
grantSql+"\r\n");
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// grantSql+"\r\n");
}
/**
@ -159,21 +163,16 @@ public class OracleExtractHelper {
+ " and t.owner||t.table_name not in (select owner||table_name from dba_tables@LINKTO"
+ oc.getName() + " where 'data_type'='CLOB')";
ResultSet rsSet = OracleConnector.getSQLExecResultSet(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
sql+"\r\n");
// FileOperateHelper
// .fileWrite(
// Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
// sql+"\r\n");
try {
while (rsSet.next()) {
try {
rsSet.getRow();
String resultSql = rsSet.getString(1).replace(";", "");
OracleConnector.execOracleSQL(conn, resultSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName(),
resultSql+"\r\n");
rsSet.getRow();
String resultSql = rsSet.getString(1).replace(";", "");
OracleConnector.execUpdateOracleSQL(conn, resultSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} catch (SQLException e) {
new CustomException(Custom4exception.threadVolume_Oracle_Except, e, rsSet.getString(1));
}

38
src/com/platform/service/impl/LogReadServiceImpl.java
Unescape View File

@ -1,19 +1,19 @@
package com.platform.service.impl;
import org.springframework.stereotype.Service;
import com.platform.service.ILogRead;
import com.platform.utils.Configs;
import com.platform.utils.FileOperateHelper;
@Service(value = "logReadService")
public class LogReadServiceImpl implements ILogRead {
@Override
public String readLog(String filename) throws Exception {
String result = FileOperateHelper.fileReader(Configs.EXTRACT_LOG_LOCALTION + filename + ".log");
return result;
}
}
package com.platform.service.impl;
import org.springframework.stereotype.Service;
import com.platform.service.ILogRead;
import com.platform.utils.Configs;
import com.platform.utils.FileOperateHelper;
@Service(value = "logReadService")
public class LogReadServiceImpl implements ILogRead {
@Override
public String readLog(String filename) throws Exception {
String result = FileOperateHelper.fileReader(Configs.EXTRACT_LOG_LOCALTION + "J" + filename.replace("-", "_")+".log");
return result;
}
}

2
src/com/platform/service/impl/OracleExtractServiceImpl.java
Unescape View File

@ -69,7 +69,7 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
Configs.CONSOLE_LOGGER.info(sb.toString());
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "1"); //更新oracle汇总状态，0标示为未汇总，1标示汇总中，2标示汇总完成
oracleExtract.createDBLink(conn, collectOracle); //创建dblink
oracleExtract.createTableSpace(conn, oracleModel); //创建表空间
oracleExtract.createTableSpace(conn, collectOracle, oracleModel); //创建表空间
oracleExtract.createUser(conn, collectOracle, oracleModel);//创建用户并授权
oracleExtract.extractColleDB(conn, collectOracle);//执行抽取
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "2"); //更新oracle汇总状态，0标示为未汇总，1标示汇总中，2标示汇总完成

15
src/com/platform/utils/FileOperateHelper.java
Unescape View File

@ -29,12 +29,13 @@ public class FileOperateHelper {
return;
}
try {
path = path+".log";
File file = new File(path);
if (file.exists())
file.createNewFile();
FileOutputStream out = new FileOutputStream(file, true); // 如果追加方式用true
StringBuffer sb = new StringBuffer();
sb.append(message).append("\n");
sb.append(message);
out.write(sb.toString().getBytes("utf-8"));
} catch (IOException e) {
// TODO: handle exception
@ -47,20 +48,20 @@ public class FileOperateHelper {
* @return
*/
@SuppressWarnings("resource")
public static String fileReader(String path) {
public static String fileReader(String path) {
StringBuffer sb = new StringBuffer();
String tempString = "";
try {
File file = new File(path);
File file = new File(path);
if (!file.exists())
return "";
return "当前没有日志信息！";
FileInputStream fis = new FileInputStream(file);
BufferedReader br = new BufferedReader(new InputStreamReader(fis));
BufferedReader br = new BufferedReader(new InputStreamReader(fis,"UTF-8"));
while ((tempString = br.readLine()) != null) {
sb.append(tempString);
sb.append(tempString).append("\r\n");
}
} catch (Exception e) {
// TODO: handle exception
Configs.CONSOLE_LOGGER.info(e.getMessage());
}
return sb.toString();
}

Write Preview
Loading…
Cancel
Save