--修复抽取bug

web_backend_develope
wu ming 8 years ago
parent bf3fcb267f
commit f08b57e3d7

@ -83,30 +83,30 @@ oracle-psw=oracle
# windows path # windows path
#============================================================================================================= #=============================================================================================================
#extract-log-localtion=D:\\test\\log\\ extract-log-localtion=D:\\test\\log\\
#extract-standard-log-localtion=D:\\test\\log2\\ extract-standard-log-localtion=D:\\test\\log2\\
#
#file_upload_path=D:\\test\\
#file_download_path=D:\\test\\export.xlsx
#
#package_download_path=D:\\test\\
#package_name=sql_script_standard
#
#sql_script_path_last=D:\\test\\sql_script_last\\
#sql_script_path_standard=D:\\test\\sql_script_standard\\
#============================================================================================================= file_upload_path=D:\\test\\
# linux path file_download_path=D:\\test\\export.xlsx
#=============================================================================================================
extract-log-localtion=/home/web_manage/log/ package_download_path=D:\\test\\
extract-standard-log-localtion=/home/web_manage/log2/ package_name=sql_script_standard
file_upload_path=/excel_import_dir/ sql_script_path_last=D:\\test\\sql_script_last\\
file_download_path=/excel_export_dir/export.xlsx sql_script_path_standard=D:\\test\\sql_script_standard\\
package_download_path=/ #=============================================================================================================
package_name=DefaultDescription # linux path
#=============================================================================================================
sql_script_path_last=/DefaultDescription_last/ #extract-log-localtion=/home/web_manage/log/
sql_script_path_standard=/DefaultDescription/ #extract-standard-log-localtion=/home/web_manage/log2/
#
#file_upload_path=/excel_import_dir/
#file_download_path=/excel_export_dir/export.xlsx
#
#package_download_path=/
#package_name=DefaultDescription
#
#sql_script_path_last=/DefaultDescription_last/
#sql_script_path_standard=/DefaultDescription/

@ -23,28 +23,30 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import com.base.BaseController; import com.base.BaseController;
import com.platform.entities.BasedTask;
import com.platform.entities.DataInfoEntity; import com.platform.entities.DataInfoEntity;
import com.platform.entities.DataInfoEntityMoveTmp; import com.platform.entities.DataInfoEntityMoveTmp;
import com.platform.entities.GatherOracleInfo; import com.platform.entities.GatherOracleInfo;
import com.platform.entities.OracleConnectorParams;
import com.platform.entities.OracleExtractExecuter;
import com.platform.form.PagerOptions; import com.platform.form.PagerOptions;
import com.platform.form.oracleForm; import com.platform.form.oracleForm;
import com.platform.form.volumeMoveForm; import com.platform.form.volumeMoveForm;
import com.platform.http.HttpUtils;
import com.platform.service.DataInfoService; import com.platform.service.DataInfoService;
import com.platform.service.ICodeService; import com.platform.service.ICodeService;
import com.platform.service.IGfsService; import com.platform.service.IGatherOracleService;
import com.platform.service.ILogRead; import com.platform.service.ILogRead;
import com.platform.service.IMoveDataService; import com.platform.service.IMoveDataService;
import com.platform.service.IGatherOracleService;
import com.platform.service.IOracleExtractService; import com.platform.service.IOracleExtractService;
import com.platform.service.IVolumeService; import com.platform.service.IVolumeService;
import com.platform.service.OracleExtractTask;
import com.platform.service.OracleStatusService; import com.platform.service.OracleStatusService;
import com.platform.service.thread.ThreadExtractOracle;
import com.platform.service.thread.ThreadGainOracleConnect;
import com.platform.utils.Constant; import com.platform.utils.Constant;
import com.platform.utils.UtilsHelper; import com.platform.utils.UtilsHelper;
/** /**
*
*
* @author chen * @author chen
* *
*/ */
@ -99,7 +101,9 @@ public class DataModelController extends BaseController {
this.dfs = dfs; this.dfs = dfs;
} }
/** -- /**
* --
*
* @param res * @param res
* @param req * @param req
* @return * @return
@ -124,17 +128,19 @@ public class DataModelController extends BaseController {
sb.append(str).append(":").append("null").append(","); sb.append(str).append(":").append("null").append(",");
} }
} }
log.info(sb.deleteCharAt(sb.length() - 1) log.info(sb.deleteCharAt(sb.length() - 1).append("}").toString());
.append("}").toString());
PagerOptions pagerOptions = (PagerOptions) UtilsHelper PagerOptions pagerOptions = (PagerOptions) UtilsHelper
.newObjAndSetAttrsByClass(PagerOptions.class, params); .newObjAndSetAttrsByClass(PagerOptions.class, params);
pagerOptions.setCurrentPageNum(Integer.valueOf(params.get("currentPageNum"))); pagerOptions.setCurrentPageNum(Integer.valueOf(params
.get("currentPageNum")));
// 冷热区查询字段mark // 冷热区查询字段mark
pagerOptions.setMark(pagerOptions.getVolumeType().trim()); pagerOptions.setMark(pagerOptions.getVolumeType().trim());
return dfs.getPagerTableData(pagerOptions); return dfs.getPagerTableData(pagerOptions);
} }
/** -- /**
* --
*
* @param res * @param res
* @param req * @param req
* @throws Exception * @throws Exception
@ -149,7 +155,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** oracle /**
* oracle
*
* @param res * @param res
* @param req * @param req
* @throws UnsupportedEncodingException * @throws UnsupportedEncodingException
@ -165,7 +173,8 @@ public class DataModelController extends BaseController {
log.info("执行连接\t" + rcName); log.info("执行连接\t" + rcName);
String cmd = "kubectl label --overwrite rc " + rcName String cmd = "kubectl label --overwrite rc " + rcName
+ " status=0"; + " status=0";
List<String> rList = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); List<String> rList = Constant.ganymedSSH
.execCmdWaitAcquiescent(cmd);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
for (String string : rList) for (String string : rList)
sb.append(string).append("\n"); sb.append(string).append("\n");
@ -175,7 +184,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** oracle /**
* oracle
*
* @param res * @param res
* @param req * @param req
* @throws UnsupportedEncodingException * @throws UnsupportedEncodingException
@ -196,7 +207,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** gfs volume /**
* gfs volume
*
* @return * @return
* @throws Exception * @throws Exception
*/ */
@ -208,7 +221,9 @@ public class DataModelController extends BaseController {
return rest; return rest;
} }
/** /**
*
*
* @param res * @param res
* @param req * @param req
* @param form * @param form
@ -233,7 +248,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** oracle /**
* oracle
*
* @param res * @param res
* @param req * @param req
* @param form * @param form
@ -243,37 +260,30 @@ public class DataModelController extends BaseController {
public void oracleExtract(HttpServletRequest res, HttpServletResponse req, public void oracleExtract(HttpServletRequest res, HttpServletResponse req,
@RequestBody oracleForm form) throws Exception { @RequestBody oracleForm form) throws Exception {
log.error("/oracle/{name}/extract"); log.error("/oracle/{name}/extract");
boolean isConnect = false; if (null != form.getTarget()) { // 检查请求参数中是否包含汇总库信息
//5秒内是否能获得oracle连接否则认为超时。 boolean isConnect = OracleExtract.isConnectTotalOracle(form
if (null != form.getTarget()) { .getTarget()); // 检查汇总库是否可以连接成功连接成功返回200状态吗连接失败返回500状态吗
ThreadGainOracleConnect thOrcl = new ThreadGainOracleConnect(form, OracleExtract);
thOrcl.start();
for (int i = 0; i < 10; i++) {
Thread.sleep(400);
isConnect = thOrcl.isConnect();
if (isConnect) { if (isConnect) {
break; req.setStatus(200);
} if (null != form.getInneed() && form.getInneed().size() > 0) {
else { for (OracleConnectorParams oracleParams : form.getInneed()) {
if (thOrcl.isExcept()) { BasedTask task = new OracleExtractTask(
break; oracleParams.getName(), form.getTarget(),
} oracleParams, OracleExtract);
Thread.sleep(100); OracleExtractExecuter oee = new OracleExtractExecuter(
} task);
new Thread(oee, oracleParams.getName()).start();
} }
} }
if (isConnect) } else
req.setStatus(200);
else
req.setStatus(500); req.setStatus(500);
// 开始抽取数据到汇总库
if (isConnect && null != form.getInneed() && form.getInneed().size() > 0) {
ThreadExtractOracle thExtra = new ThreadExtractOracle(form, OracleExtract);
thExtra.start();
} }
} }
/** oracle /**
* oracle
*
* @return * @return
* @throws Exception * @throws Exception
*/ */
@ -285,7 +295,9 @@ public class DataModelController extends BaseController {
return result; return result;
} }
/** oracle /**
* oracle
*
* @param req * @param req
* @param res * @param res
* @param id * @param id
@ -301,7 +313,9 @@ public class DataModelController extends BaseController {
res.setStatus(200); res.setStatus(200);
} }
/** oracle /**
* oracle
*
* @param res * @param res
* @param req * @param req
* @param id * @param id
@ -319,7 +333,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** oracle /**
* oracle
*
* @param res * @param res
* @param req * @param req
* @param id * @param id
@ -337,7 +353,9 @@ public class DataModelController extends BaseController {
req.setStatus(200); req.setStatus(200);
} }
/** /**
*
*
* @return * @return
* @throws Exception * @throws Exception
*/ */
@ -349,7 +367,9 @@ public class DataModelController extends BaseController {
return result; return result;
} }
/** /**
*
*
* @param res * @param res
* @param req * @param req
* @param id * @param id
@ -368,7 +388,9 @@ public class DataModelController extends BaseController {
return result; return result;
} }
/** /**
*
*
* @param res * @param res
* @param req * @param req
* @param move * @param move
@ -386,7 +408,9 @@ public class DataModelController extends BaseController {
return 1; return 1;
} }
/** code /**
* code
*
* @return * @return
* @throws Exception * @throws Exception
*/ */
@ -398,7 +422,9 @@ public class DataModelController extends BaseController {
return result; return result;
} }
/** oracle /**
* oracle
*
* @param name * @param name
* @param res * @param res
* @param req * @param req

@ -0,0 +1,32 @@
package com.platform.entities;
public abstract class AbstractOracleExtractTask implements BasedTask {
private String name;
private int status; // 任务的执行状态,0未执行1执行中2完成3失败
public AbstractOracleExtractTask() {
}
public AbstractOracleExtractTask(String name) {
this.name = name;
}
public abstract void handler();
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
}

@ -0,0 +1,5 @@
package com.platform.entities;
public interface BasedTask {
public void handler();
}

@ -0,0 +1,15 @@
package com.platform.entities;
public class OracleExtractExecuter implements Runnable {
private BasedTask task;
public OracleExtractExecuter(BasedTask task) {
this.task = task;
}
@Override
public void run() {
if (null != task)
task.handler();
}
}

@ -16,7 +16,8 @@ import com.platform.utils.FileOperateHelper;
public class OracleConnector { public class OracleConnector {
public static Logger log = Configs.DAILY_ROLLING_LOGGER.getLogger(OracleConnector.class); public static Logger log = Configs.DAILY_ROLLING_LOGGER
.getLogger(OracleConnector.class);
public OracleConnector() { public OracleConnector() {
} }
@ -30,24 +31,27 @@ public class OracleConnector {
} }
} }
public synchronized static Connection connectionBuilder(String url,
public synchronized static Connection connectionBuilder(String url, String user, String user, String password, OracleConnectorParams oc)
String password, OracleConnectorParams oc) throws CustomException { throws CustomException {
Connection conn = null; Connection conn = null;
try { try {
conn = DriverManager.getConnection(url, user, password); conn = DriverManager.getConnection(url, user, password);
} catch (SQLException e) { } catch (SQLException e) {
Configs.CONSOLE_LOGGER.info("创建oracle连接失败: [" + e.getMessage() + "]"); Configs.CONSOLE_LOGGER.info("创建oracle连接失败: [" + e.getMessage()
+ "]");
if (null != oc) { if (null != oc) {
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper.fileWrite(
+ oc.getName()+".log", "创建oracle连接失败: [" + e.getMessage() + "]\r\n"); Configs.EXTRACT_LOG_LOCALTION + oc.getName() + ".log",
"创建oracle连接失败: [" + e.getMessage() + "]\r\n");
} }
throw new CustomException(Custom4exception.OracleSQL_Except, e); throw new CustomException(Custom4exception.OracleSQL_Except, e);
} }
return conn; return conn;
} }
public synchronized static boolean canConnect(String url, String user, String password) { public synchronized static boolean canConnect(String url, String user,
String password) {
Connection result = null; Connection result = null;
try { try {
result = connectionBuilder(url, user, password, null); result = connectionBuilder(url, user, password, null);
@ -57,25 +61,27 @@ public class OracleConnector {
return (null != result); return (null != result);
} }
public synchronized static ResultSet getSQLExecResultSet(Connection conn, String sql, String filePath) { public synchronized static ResultSet getSQLExecResultSet(Connection conn,
String sql, String filePath) {
ResultSet resultSet = null; ResultSet resultSet = null;
if (null != filePath) { if (null != filePath) {
filePath = filePath.replace(".log", ""); filePath = filePath.replace(".log", "");
} }
Statement statement = null; Statement statement = null;
try { try {
statement = conn statement = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE,
ResultSet.CONCUR_UPDATABLE); ResultSet.CONCUR_UPDATABLE);
resultSet = statement.executeQuery(sql); resultSet = statement.executeQuery(sql);
FileOperateHelper /*
.fileWrite(filePath+".log", sql+ "\r\n"+"OK \r\n"); * if(resultSet.next()){ System.out.println(resultSet.getInt(1)); }
*/
FileOperateHelper.fileWrite(filePath + ".log", sql + "\r\n"
+ "OK \r\n");
} catch (SQLException e) { } catch (SQLException e) {
FileOperateHelper FileOperateHelper.fileWrite(filePath + ".log",
.fileWrite(filePath+".log", sql+ "\r\n"+e.getMessage()+"\r\n"); sql + "\r\n" + e.getMessage() + "\r\n");
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} } finally {
finally{
if (null != statement) { if (null != statement) {
try { try {
statement.close(); statement.close();
@ -88,11 +94,12 @@ public class OracleConnector {
return resultSet; return resultSet;
} }
public synchronized static ResultSet getSQLExecResultSet(String url, String user, public synchronized static ResultSet getSQLExecResultSet(String url,
String password, String sql, String filePath) { String user, String password, String sql, String filePath) {
ResultSet result = null; ResultSet result = null;
try { try {
result = getSQLExecResultSet(connectionBuilder(url, user, password, null), sql, filePath); result = getSQLExecResultSet(
connectionBuilder(url, user, password, null), sql, filePath);
} catch (CustomException e) { } catch (CustomException e) {
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} }
@ -101,11 +108,13 @@ public class OracleConnector {
/** /**
* oracle * oracle
*
* @param conn * @param conn
* @param sql * @param sql
* @return true:sql * @return true:sql
*/ */
public synchronized static boolean execOracleSQL(Connection conn, String sql, String filePath) { public synchronized static boolean execOracleSQL(Connection conn,
String sql, String filePath) {
if (null != filePath) { if (null != filePath) {
filePath = filePath.replace(".log", ""); filePath = filePath.replace(".log", "");
} }
@ -113,16 +122,16 @@ public class OracleConnector {
Statement statement = null; Statement statement = null;
try { try {
statement = conn.createStatement(); statement = conn.createStatement();
flag =statement.execute(sql); statement.executeUpdate(sql);
FileOperateHelper flag = true;
.fileWrite(filePath+".log", sql+ "\r\n"+ flag +" \r\n"); FileOperateHelper.fileWrite(filePath + ".log", sql + "\r\n" + flag
+ " \r\n");
} catch (SQLException e) { } catch (SQLException e) {
flag = false; flag = false;
FileOperateHelper FileOperateHelper.fileWrite(filePath + ".log",
.fileWrite(filePath+".log", sql+ "\r\n"+e.getMessage()+"\r\n"); sql + "\r\n" + e.getMessage() + "\r\n");
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} } finally {
finally{
if (null != statement) { if (null != statement) {
try { try {
statement.close(); statement.close();
@ -134,13 +143,16 @@ public class OracleConnector {
return flag ; return flag ;
} }
/** /**
* oraclesql * oraclesql
*
* @param conn * @param conn
* @param sql * @param sql
* @return true:1 * @return true:1
*/ */
public synchronized static boolean execUpdateOracleSQL(Connection conn, String sql, String filePath) { public synchronized static boolean execUpdateOracleSQL(Connection conn,
String sql, String filePath) {
if (null != filePath) { if (null != filePath) {
filePath = filePath.replace(".log", ""); filePath = filePath.replace(".log", "");
} }
@ -148,19 +160,17 @@ public class OracleConnector {
Statement statement = null; Statement statement = null;
try { try {
statement = conn.createStatement(); statement = conn.createStatement();
if(statement.executeUpdate(sql) > 0) statement.executeUpdate(sql);
{
flag = true; flag = true;
FileOperateHelper FileOperateHelper.fileWrite(filePath + ".log", sql + "\r\n"
.fileWrite(filePath+".log", sql+ "\r\n"+"OK \r\n"); + "OK \r\n");
}
} catch (SQLException e) { } catch (SQLException e) {
flag = false; flag = false;
FileOperateHelper FileOperateHelper.fileWrite(filePath + ".log",
.fileWrite(filePath+".log", sql+ "\r\n"+e.getMessage()+"\r\n"); sql + "\r\n" + e.getMessage() + "\r\n");
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} } finally {
finally{
if (null != statement) { if (null != statement) {
try { try {
statement.close(); statement.close();

@ -33,4 +33,10 @@ public interface IOracleExtractService {
boolean extractStandardTable(String name, List<OracleConnectorParams> dataInfolist, boolean extractStandardTable(String name, List<OracleConnectorParams> dataInfolist,
GatherOracleInfo oracleConnect) throws Exception; GatherOracleInfo oracleConnect) throws Exception;
/**
*
* @param ocp
*/
public void updateDataExtractStatus(OracleConnectorParams ocp, int status);
} }

@ -16,7 +16,7 @@ import com.platform.utils.FileOperateHelper;
public class OracleExtractHelper { public class OracleExtractHelper {
public static Logger log = Configs.DAILY_ROLLING_LOGGER.getLogger(OracleExtractHelper.class); public static Logger log = Logger.getLogger(OracleExtractHelper.class);
/** /**
* dblink * dblink
@ -26,22 +26,39 @@ public class OracleExtractHelper {
* dblink * dblink
* @return * @return
*/ */
private boolean hasSameNameDBLink(Connection conn, String linkName, String filePath) { private boolean hasSameNameDBLink(Connection conn, String linkName,
String filePath) {
boolean flag = false; boolean flag = false;
String sql = "SELECT * FROM ALL_DB_LINKS WHERE DB_LINK='" + linkName+"'"; String sql = "SELECT COUNT(*) c FROM ALL_DB_LINKS WHERE DB_LINK='"
+ linkName + "'";
ResultSet rSet = null; ResultSet rSet = null;
Statement statement = null;
try { try {
rSet = OracleConnector.getSQLExecResultSet(conn, sql, null); statement = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
rSet.last(); ResultSet.CONCUR_UPDATABLE);
if (rSet.getRow() > 0) rSet = statement.executeQuery(sql);
if (rSet.next()) {
if (rSet.getInt("c") > 0)
flag = true; flag = true;
FileOperateHelper else
.fileWrite(filePath, sql+ "\r\n"+"OK \r\n"); flag = false;
}
FileOperateHelper.fileWrite(filePath + ".log", sql + "\r\n"
+ "OK \r\n");
} catch (Exception e) { } catch (Exception e) {
FileOperateHelper FileOperateHelper.fileWrite(filePath, sql + "\r\n" + e.getMessage()
.fileWrite(filePath, sql+ "\r\n"+ e.getMessage() + "\r\n连接异常 \r\n"); + "\r\n连接异常 \r\n");
log.error(Custom4exception.threadVolume_Oracle_Except, e); log.error(Custom4exception.threadVolume_Oracle_Except, e);
} finally {
if (null != statement) {
try {
statement.close();
} catch (SQLException e) {
log.error(Custom4exception.OracleSQL_Except, e);
}
}
} }
return flag; return flag;
} }
@ -49,12 +66,13 @@ public class OracleExtractHelper {
/** /**
* dblink * dblink
* *
* @param conn * @param conn
* @param oc *
* dblink * @param oc
* dblink
*/ */
public void createDBLink(Connection conn, OracleConnectorParams oc) { public void createDBLink(Connection conn, OracleConnectorParams oc) {
String linkName = "LINKTO" + oc.getName(); String linkName = "LINKTO_J" + oc.getName().replaceAll("-", "_");
String sql = "CREATE PUBLIC DATABASE LINK " String sql = "CREATE PUBLIC DATABASE LINK "
+ linkName + linkName
+ " CONNECT TO " + " CONNECT TO "
@ -66,24 +84,30 @@ public class OracleExtractHelper {
+ ")))(CONNECT_DATA =(SERVICE_NAME =" + oc.getDatabaseName() + ")))(CONNECT_DATA =(SERVICE_NAME =" + oc.getDatabaseName()
+ ")))\'"; + ")))\'";
if (null != oc) { if (null != oc) {
if (hasSameNameDBLink(conn, linkName, Configs.EXTRACT_LOG_LOCALTION + oc.getName()+".log")) { // 如果dblink已经存在,先删除dblink在创建dblink if (hasSameNameDBLink(conn, linkName, Configs.EXTRACT_LOG_LOCALTION
String deleteSql = "DROP PUBLIC DATABASE LINK " + oc.getName())) { // 如果dblink已经存在,先删除dblink在创建dblink
+ linkName; String deleteSql = "DROP PUBLIC DATABASE LINK " + linkName;
// 删除 dblink // 删除 dblink
if (OracleConnector.execOracleSQL(conn, deleteSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())) { if (OracleConnector.execOracleSQL(conn, deleteSql,
if(!OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())){ Configs.EXTRACT_LOG_LOCALTION + oc.getName())) {
OracleConnector.execUpdateOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); if (!OracleConnector.execOracleSQL(conn, sql,
Configs.EXTRACT_LOG_LOCALTION + oc.getName())) {
OracleConnector.execUpdateOracleSQL(conn, sql,
Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} }
} else { } else {
Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!"); Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!");
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
+ oc.getName()+".log", "删除已有的DBLink失败,无法创建新的DBLink!"+"\r\n"); + oc.getName() + ".log",
"删除已有的DBLink失败,无法创建新的DBLink!" + "\r\n");
} }
} else { } else {
// 否则创建dblink // 否则创建dblink
if(!OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName())){ if (!OracleConnector.execOracleSQL(conn, sql,
OracleConnector.execUpdateOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); Configs.EXTRACT_LOG_LOCALTION + oc.getName())) {
OracleConnector.execUpdateOracleSQL(conn, sql,
Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} }
} }
} }
@ -92,52 +116,66 @@ public class OracleExtractHelper {
/** /**
* *
* *
* @param conn * @param conn
*
* @param collectOracle * @param collectOracle
* @param oc * @param oc
*
* @return * @return
*/ */
public boolean createTableSpace(Connection conn, OracleConnectorParams collectOracle, GatherOracleInfo oc) { public boolean createTableSpace(Connection conn,
OracleConnectorParams collectOracle, GatherOracleInfo oc) {
String tmpSql = "select TABLESPACE_NAME from dba_tablespaces where TABLESPACE_NAME = '" String tmpSql = "select TABLESPACE_NAME from dba_tablespaces where TABLESPACE_NAME = '"
+ oc.getTableName() + "'"; + oc.getTableName() + "'";
// 存在 表空间 // 存在 表空间
if (OracleConnector.execUpdateOracleSQL(conn, tmpSql, Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName())) { if (OracleConnector.execUpdateOracleSQL(conn, tmpSql,
Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName())) {
return true; return true;
} else { } else {
String sql = "create tablespace " + oc.getTableName() String sql = "create tablespace " + oc.getTableName()
+ " datafile '" + Configs.GATHER_TABLESPACE_PATH + " datafile '" + Configs.GATHER_TABLESPACE_PATH
+ oc.getTableName() + ".dbf" + oc.getTableName() + ".dbf"
+ "' size 512M autoextend on next 512M maxsize unlimited"; + "' size 512M autoextend on next 512M maxsize unlimited";
return OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName()); return OracleConnector.execOracleSQL(conn, sql,
Configs.EXTRACT_LOG_LOCALTION + collectOracle.getName());
} }
} }
/** /**
* *
* *
* @param conn * @param conn
* @param oc *
* @param oc
*
*/ */
public void createUser(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) { public void createUser(Connection conn, OracleConnectorParams oc,
String strTUser = oc.getName() + totalOracle.getSuffix(); GatherOracleInfo totalOracle) {
String strTUser = "J" + oc.getName().replaceAll("-", "_")
+ totalOracle.getSuffix();
String sql = "Create user " + strTUser + " default tablespace " String sql = "Create user " + strTUser + " default tablespace "
+ totalOracle.getTableName() + " identified by " + totalOracle.getTableName() + " identified by "
+ Configs.GATHER_TABLE_PASSWORD; + Configs.GATHER_TABLE_PASSWORD;
String grantSql = "grant connect, resource, dba to " + strTUser; String grantSql = "grant connect, resource, dba to " + strTUser;
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION
+ oc.getName());
OracleConnector.execOracleSQL(conn, grantSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, grantSql,
Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} }
/** /**
* *
* *
* @param conn * @param conn
* @param oc *
* @param oc
*
*/ */
public void createOnlyUser(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) { public void createOnlyUser(Connection conn, OracleConnectorParams oc,
GatherOracleInfo totalOracle) {
String strTUser = Configs.GATHER_STANDARD_USER_NAME; String strTUser = Configs.GATHER_STANDARD_USER_NAME;
String sql = "Create user " + strTUser + " default tablespace " String sql = "Create user " + strTUser + " default tablespace "
@ -145,18 +183,23 @@ public class OracleExtractHelper {
+ Configs.GATHER_TABLE_PASSWORD; + Configs.GATHER_TABLE_PASSWORD;
String grantSql = "grant connect, resource, dba to " + strTUser; String grantSql = "grant connect, resource, dba to " + strTUser;
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, sql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
OracleConnector.execOracleSQL(conn, grantSql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, grantSql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
} }
/** /**
* *
* *
* @param conn * @param conn
* @param oc *
* @param oc
*
*/ */
public void createTable(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) { public void createTable(Connection conn, OracleConnectorParams oc,
GatherOracleInfo totalOracle) {
String strTUser = oc.getName() + totalOracle.getSuffix(); String strTUser = oc.getName() + totalOracle.getSuffix();
String sql = "Create table " + strTUser + " default tablespace " String sql = "Create table " + strTUser + " default tablespace "
@ -164,36 +207,54 @@ public class OracleExtractHelper {
+ Configs.GATHER_TABLE_PASSWORD; + Configs.GATHER_TABLE_PASSWORD;
String grantSql = "grant connect, resource, dba to " + strTUser; String grantSql = "grant connect, resource, dba to " + strTUser;
OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_LOG_LOCALTION
+ oc.getName());
OracleConnector.execOracleSQL(conn, grantSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, grantSql,
Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} }
/** /**
* *
* *
* @param conn * @param conn
* @param oc *
* @param oc
*
*/ */
public void extractColleDB(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) { public void extractColleDB(Connection conn, OracleConnectorParams oc,
String strTUser = oc.getName() + totalOracle.getSuffix(); GatherOracleInfo totalOracle) {
String strTUser = "J" + oc.getName().replaceAll("-", "_")
+ totalOracle.getSuffix();
String sql = "SELECT 'create table " String sql = "SELECT 'create table "
+ strTUser + strTUser
+ ".J'|| substr(t.OWNER||'_'||t.TABLE_NAME,0,29)||' as select * from '||t.OWNER||'.'||t.TABLE_NAME||" + ".J'|| substr(t.OWNER||'_'||t.TABLE_NAME,0,29)||' as select * from '||t.OWNER||'.'||t.TABLE_NAME||"
+ "'@LINKTO" + "'@LINKTO_J"
+ oc.getName() + oc.getName().replaceAll("-", "_")
+ ";' FROM dba_tables@LINKTO" + ";' FROM dba_tables@LINKTO_J"
+ oc.getName() + oc.getName().replaceAll("-", "_")
+ " t WHERE t.TABLESPACE_NAME NOT IN ('SYSTEM','SYSAUX')" + " t WHERE t.TABLESPACE_NAME NOT IN ('SYSTEM','SYSAUX')"
+ " and t.owner||t.table_name not in (select owner||table_name from dba_tables@LINKTO" + " and t.owner||t.table_name not in (select owner||table_name from dba_tables@LINKTO_J"
+ oc.getName() + " where 'data_type'='CLOB')"; + oc.getName().replaceAll("-", "_")
ResultSet rsSet = OracleConnector.getSQLExecResultSet(conn, sql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); + " where 'data_type'='CLOB')";
/*
* ResultSet rsSet = OracleConnector.getSQLExecResultSet(conn, sql,
* Configs.EXTRACT_LOG_LOCALTION + oc.getName());
*/
Statement statement = null;
try { try {
while (rsSet.next()) { statement = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
ResultSet.CONCUR_UPDATABLE);
ResultSet resultSet = statement.executeQuery(sql);
FileOperateHelper.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName() + ".log", sql
+ "\r\n" + "OK \r\n");
while (resultSet.next()) {
try { try {
rsSet.getRow(); String resultSql = resultSet.getString(1).replace(";", "");
String resultSql = rsSet.getString(1).replace(";", ""); OracleConnector.execUpdateOracleSQL(conn, resultSql,
OracleConnector.execUpdateOracleSQL(conn, resultSql, Configs.EXTRACT_LOG_LOCALTION + oc.getName()); Configs.EXTRACT_LOG_LOCALTION + oc.getName());
} catch (SQLException e) { } catch (SQLException e) {
log.error(Custom4exception.threadVolume_Oracle_Except, e); log.error(Custom4exception.threadVolume_Oracle_Except, e);
} }
@ -204,7 +265,9 @@ public class OracleExtractHelper {
} }
/** /**
*
*
* @return true * @return true
*/ */
public boolean testConnect(Connection conn) { public boolean testConnect(Connection conn) {
@ -224,20 +287,27 @@ public class OracleExtractHelper {
/** /**
* -- * --
* *
* @param conn * @param conn
* @param oc *
* @param oc
*
*/ */
public void extractStandardPayTable(Connection conn, OracleConnectorParams oc, GatherOracleInfo totalOracle) { public void extractStandardPayTable(Connection conn,
OracleConnectorParams oc, GatherOracleInfo totalOracle) {
String strTUser = Configs.GATHER_STANDARD_USER_NAME; String strTUser = Configs.GATHER_STANDARD_USER_NAME;
createPay(conn, oc); createPay(conn, oc);
String sql = "insert into " + strTUser + "." + Configs.GATHER_STANDARD_PAY_TABLE_NAME + " select * from " String sql = "insert into " + strTUser + "."
+ strTUser + "." + Configs.GATHER_STANDARD_PAY_TABLE_NAME +"@LINKTOST" + Configs.GATHER_STANDARD_PAY_TABLE_NAME + " select * from "
+ oc.getName(); + strTUser + "." + Configs.GATHER_STANDARD_PAY_TABLE_NAME
+ "@LINKTOST" + oc.getName();
String resultSql = sql.replace(";", ""); String resultSql = sql.replace(";", "");
OracleConnector.execOracleSQL(conn, resultSql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, resultSql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
} }
/** -- /**
* --
*
* @param conn * @param conn
* @param collectOracle * @param collectOracle
* @param oracleModel * @param oracleModel
@ -247,16 +317,19 @@ public class OracleExtractHelper {
createExec(conn, collectOracle); createExec(conn, collectOracle);
String strTUser = Configs.GATHER_STANDARD_USER_NAME; String strTUser = Configs.GATHER_STANDARD_USER_NAME;
String sql = "insert into " + strTUser + "." + Configs.GATHER_STANDARD_EXEC_TABLE_NAME + " select * from " String sql = "insert into " + strTUser + "."
+ strTUser + "." + Configs.GATHER_STANDARD_EXEC_TABLE_NAME +"@LINKTOST" + Configs.GATHER_STANDARD_EXEC_TABLE_NAME + " select * from "
+ collectOracle.getName(); + strTUser + "." + Configs.GATHER_STANDARD_EXEC_TABLE_NAME
+ "@LINKTOST" + collectOracle.getName();
String resultSql = sql.replace(";", ""); String resultSql = sql.replace(";", "");
OracleConnector.execOracleSQL(conn, resultSql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + collectOracle.getName()); OracleConnector.execOracleSQL(
conn,
resultSql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ collectOracle.getName());
} }
private void createPay(Connection conn, OracleConnectorParams oc) { private void createPay(Connection conn, OracleConnectorParams oc) {
String payCmd = "CREATE TABLE u_bzbjy.zfxxb(XZQHDM Varchar(255),XZQHMC Varchar(255),PZBH Varchar(255),LYZBKZH Varchar(255)," String payCmd = "CREATE TABLE u_bzbjy.zfxxb(XZQHDM Varchar(255),XZQHMC Varchar(255),PZBH Varchar(255),LYZBKZH Varchar(255),"
+ "ZFDATE Varchar(255),YSDWCODE Varchar(255),YSDWNAME Varchar(255),YWGKCS Varchar(255),XMCODE Varchar(255),XMNAME Varchar(255)," + "ZFDATE Varchar(255),YSDWCODE Varchar(255),YSDWNAME Varchar(255),YWGKCS Varchar(255),XMCODE Varchar(255),XMNAME Varchar(255),"
@ -266,7 +339,8 @@ public class OracleExtractHelper {
+ "QSZHNAME Varchar(255),QSYHCODE Varchar(255),QSYHNAME Varchar(255),JE Numeric(18,2), SFTK Varchar(255),NIAN Varchar(255),ZY Varchar(255))"; + "QSZHNAME Varchar(255),QSYHCODE Varchar(255),QSYHNAME Varchar(255),JE Numeric(18,2), SFTK Varchar(255),NIAN Varchar(255),ZY Varchar(255))";
try { try {
OracleConnector.execOracleSQL(conn, payCmd, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, payCmd,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
} catch (Exception e) { } catch (Exception e) {
log.error(Custom4exception.threadVolume_Oracle_Except, e); log.error(Custom4exception.threadVolume_Oracle_Except, e);
} }
@ -275,9 +349,10 @@ public class OracleExtractHelper {
/** /**
* dblink * dblink
* *
* @param conn * @param conn
* @param oc *
* dblink * @param oc
* dblink
*/ */
public void createStandardDBLink(Connection conn, OracleConnectorParams oc) { public void createStandardDBLink(Connection conn, OracleConnectorParams oc) {
String linkName = "LINKTOST" + oc.getName(); String linkName = "LINKTOST" + oc.getName();
@ -292,26 +367,45 @@ public class OracleExtractHelper {
+ ")))(CONNECT_DATA =(SERVICE_NAME =" + oc.getDatabaseName() + ")))(CONNECT_DATA =(SERVICE_NAME =" + oc.getDatabaseName()
+ ")))\'"; + ")))\'";
if (null != oc) { if (null != oc) {
if (hasSameNameDBLink(conn, linkName, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()+".log")) { // 如果dblink已经存在,先删除dblink在创建dblink if (hasSameNameDBLink(conn, linkName,
String deleteSql = "DROP PUBLIC DATABASE LINK " Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()
+ linkName; + ".log")) { // 如果dblink已经存在,先删除dblink在创建dblink
String deleteSql = "DROP PUBLIC DATABASE LINK " + linkName;
// 删除 dblink // 删除 dblink
if (OracleConnector.execOracleSQL(conn, deleteSql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName())) { if (OracleConnector.execOracleSQL(conn, deleteSql,
// OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName())) {
if(!OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName())){ // OracleConnector.execOracleSQL(conn, sql,
OracleConnector.execUpdateOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); // Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
if (!OracleConnector.execOracleSQL(
conn,
sql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ oc.getName())) {
OracleConnector.execUpdateOracleSQL(
conn,
sql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ oc.getName());
} }
} else { } else {
Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!"); Configs.CONSOLE_LOGGER.error("删除已有的DBLink失败,无法创建新的DBLink!");
FileOperateHelper.fileWrite(Configs.EXTRACT_STANDARD_LOG_LOCALTION FileOperateHelper.fileWrite(
+ oc.getName()+".log", "删除已有的DBLink失败,无法创建新的DBLink!"+"\r\n"); Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ oc.getName() + ".log",
"删除已有的DBLink失败,无法创建新的DBLink!" + "\r\n");
} }
} else { } else {
// 否则创建dblink // 否则创建dblink
// OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); // OracleConnector.execOracleSQL(conn, sql,
if(!OracleConnector.execOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName())){ // Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
OracleConnector.execUpdateOracleSQL(conn, sql, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); if (!OracleConnector.execOracleSQL(conn, sql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName())) {
OracleConnector.execUpdateOracleSQL(
conn,
sql,
Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ oc.getName());
} }
} }
} }
@ -326,7 +420,8 @@ public class OracleExtractHelper {
+ "ZBKYJE Numeric(18,2),ZYZFBZ Varchar(255),BZ Varchar(255))"; + "ZBKYJE Numeric(18,2),ZYZFBZ Varchar(255),BZ Varchar(255))";
try { try {
OracleConnector.execOracleSQL(conn, execCmd, Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName()); OracleConnector.execOracleSQL(conn, execCmd,
Configs.EXTRACT_STANDARD_LOG_LOCALTION + oc.getName());
} catch (Exception e) { } catch (Exception e) {
log.error(Custom4exception.threadVolume_Oracle_Except, e); log.error(Custom4exception.threadVolume_Oracle_Except, e);
} }

@ -0,0 +1,107 @@
package com.platform.service;
import java.sql.Connection;
import java.util.Date;
import java.util.List;
import org.apache.log4j.Logger;
import com.base.Custom4exception;
import com.platform.entities.AbstractOracleExtractTask;
import com.platform.entities.GatherOracleInfo;
import com.platform.entities.OracleConnectorParams;
import com.platform.oracle.OracleConnector;
import com.platform.utils.Configs;
import com.platform.utils.Constant;
import com.platform.utils.DateForm;
import com.platform.utils.FileOperateHelper;
public class OracleExtractTask extends AbstractOracleExtractTask {
public static Logger log = Logger.getLogger(OracleExtractTask.class);
private GatherOracleInfo gatherOracleInfo;
private OracleConnectorParams oc;
private IOracleExtractService OracleExtract;
public OracleExtractTask(String name, GatherOracleInfo gatherOracleInfo,
OracleConnectorParams oc, IOracleExtractService OracleExtract) {
super(name);
this.gatherOracleInfo = gatherOracleInfo;
this.oc = oc;
this.OracleExtract = OracleExtract;
}
public OracleExtractTask(GatherOracleInfo gatherOracleInfo,
OracleConnectorParams oc, IOracleExtractService OracleExtract) {
this.gatherOracleInfo = gatherOracleInfo;
this.oc = oc;
this.OracleExtract = OracleExtract;
}
private OracleExtractHelper oracleExtract = new OracleExtractHelper();
@Override
public void handler() {
// TODO Auto-generated method stub
try {
Connection conn = OracleConnector.connectionBuilder(
createConnectUrl(), gatherOracleInfo.getUser(),
gatherOracleInfo.getPassword(), oc);
if (null != conn) {
String cmd = "kubectl label --overwrite rc " + oc.getName()
+ " isExtract=1";
OracleExtract.updateDataExtractStatus(oc, 1); // 更新数据库的状态
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName()
+ ".log",
"["
+ DateForm
.date2StringBysecond(new Date())
+ "]>>>>>>>>>>>>>>>>开始汇总 >>>>>>>>>>>>>>>>>>\r\n");
List<String> rList = Constant.ganymedSSH
.execCmdWaitAcquiescent(cmd);
StringBuffer sb = new StringBuffer();
for (String string : rList)
sb.append(string).append("\n");
Configs.CONSOLE_LOGGER.info(sb.toString());
oracleExtract.createDBLink(conn, oc); // 创建dblink
oracleExtract.createTableSpace(conn, oc, gatherOracleInfo); // 创建表空间
oracleExtract.createUser(conn, oc, gatherOracleInfo);// 创建用户并授权
oracleExtract.extractColleDB(conn, oc, gatherOracleInfo);// 执行抽取
cmd = "kubectl label --overwrite rc " + oc.getName()
+ " isExtract=2";
rList = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
sb = new StringBuffer();
for (String string : rList)
sb.append(string).append("\n");
Configs.CONSOLE_LOGGER.info(sb.toString());
FileOperateHelper
.fileWrite(
Configs.EXTRACT_LOG_LOCALTION + oc.getName()
+ ".log",
"["
+ DateForm
.date2StringBysecond(new Date())
+ "]>>>>>>>>>>>>>>>>汇总结束 >>>>>>>>>>>>>>>>>>\r\n\r\n");
OracleExtract.updateDataExtractStatus(oc, 2); // 更新数据库的状态
}
} catch (Exception e) {
log.error(Custom4exception.OracleSQL_Except, e);
}
}
private String createConnectUrl() {
String answer = "";
if (null != gatherOracleInfo) {
answer = "jdbc:oracle:thin:@" + gatherOracleInfo.getIp() + ":"
+ gatherOracleInfo.getPort() + ":"
+ gatherOracleInfo.getDatabaseName();
}
return answer;
}
}

@ -12,7 +12,7 @@ public class LogReadServiceImpl implements ILogRead {
@Override @Override
public String readLog(String filename) throws Exception { public String readLog(String filename) throws Exception {
String result = FileOperateHelper.fileReader(Configs.EXTRACT_LOG_LOCALTION + "J" + filename.replace("-", "_")+".log"); String result = FileOperateHelper.fileReader(Configs.EXTRACT_LOG_LOCALTION + filename+".log");
return result; return result;
} }

@ -31,7 +31,8 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
/** /**
* *
*/ */
public final static Logger log = Logger.getLogger(OracleExtractServiceImpl.class); public final static Logger log = Logger
.getLogger(OracleExtractServiceImpl.class);
@Resource(name = "dataInfoDao") @Resource(name = "dataInfoDao")
private DataInfoDao dataInfoDao; private DataInfoDao dataInfoDao;
@ -51,52 +52,74 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
private OracleConnector connect = new OracleConnector(); private OracleConnector connect = new OracleConnector();
@Override @Override
public boolean extractOracle(String name, List<OracleConnectorParams> dataInfolist, public boolean extractOracle(String name,
GatherOracleInfo oracleConnect) throws Exception { List<OracleConnectorParams> datainfos, GatherOracleInfo oracleModel)
throws Exception {
boolean isSuccess = false; boolean isSuccess = false;
try { try {
// map转 bean(汇总库信息-带tableName的) // map转 bean(汇总库信息-带tableName的)
GatherOracleInfo oracleModel = oracleConnect; // GatherOracleInfo oracleModel = oracleConnect;
// 采集库连接参数 // 采集库连接参数
List<OracleConnectorParams> datainfos = dataInfolist; // List<OracleConnectorParams> datainfos = dataInfolist;
if (datainfos.size() == 0) { if (datainfos.size() == 0) {
return false; return false;
} }
Connection conn = OracleConnector.connectionBuilder("jdbc:oracle:thin:@" + oracleModel.getIp() + ":" + oracleModel.getPort() + ":" Connection conn = OracleConnector.connectionBuilder(
+ oracleModel.getDatabaseName(), oracleModel.getUser(), oracleModel.getPassword(),dataInfolist.get(0)); "jdbc:oracle:thin:@" + oracleModel.getIp() + ":"
+ oracleModel.getPort() + ":"
+ oracleModel.getDatabaseName(),
oracleModel.getUser(), oracleModel.getPassword(),
datainfos.get(0));
if (null == conn) { if (null == conn) {
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
+ dataInfolist.get(0).getName()+".log", "创建oracle连接失败: [" + conn + "]\r\n"); + datainfos.get(0).getName() + ".log",
"创建oracle连接失败: [" + conn + "]\r\n");
return false; return false;
} }
for (OracleConnectorParams collectOracle : datainfos) { for (OracleConnectorParams collectOracle : datainfos) {
String replicasName = collectOracle.getName(); String replicasName = collectOracle.getName();
try { try {
if(null != collectOracle.getDataId() && !"".equals(collectOracle.getDataId())){ if (null != collectOracle.getDataId()
&& !"".equals(collectOracle.getDataId())) {
DataInfoEntity data = new DataInfoEntity(); DataInfoEntity data = new DataInfoEntity();
data.setId(Integer.valueOf(collectOracle.getDataId())); data.setId(Integer.valueOf(collectOracle.getDataId()));
data.setExtractStatus(1); data.setExtractStatus(1);
dataInfoDao.updateExtract(data); dataInfoDao.updateExtract(data);
collectOracle.setName("J" + collectOracle.getName().replace("-", "_")); collectOracle.setName("J"
+ collectOracle.getName().replace("-", "_"));
String cmd = "kubectl label --overwrite rc " String cmd = "kubectl label --overwrite rc "
+ replicasName + " isExtract=1"; + replicasName + " isExtract=1";
// sql日志记录时间 // sql日志记录时间
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper
+ collectOracle.getName()+".log", "\r\n 开始汇总 \r\n"+ DateForm.date2StringBysecond(new Date()) +"\r\n"); .fileWrite(
Configs.EXTRACT_LOG_LOCALTION
+ collectOracle.getName()
+ ".log",
"\r\n 开始汇总 \r\n"
+ DateForm
.date2StringBysecond(new Date())
+ "\r\n");
List<String> rList = Constant.ganymedSSH List<String> rList = Constant.ganymedSSH
.execCmdWaitAcquiescent(cmd); .execCmdWaitAcquiescent(cmd);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
for (String string : rList) for (String string : rList)
sb.append(string).append("\n"); sb.append(string).append("\n");
Configs.CONSOLE_LOGGER.info(sb.toString()); Configs.CONSOLE_LOGGER.info(sb.toString());
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "1"); //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成 // client.updateOrAddReplicasLabelById(collectOracle.getName(),
// "isExtract", "1");
// //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成
oracleExtract.createDBLink(conn, collectOracle); // 创建dblink oracleExtract.createDBLink(conn, collectOracle); // 创建dblink
oracleExtract.createTableSpace(conn, collectOracle, oracleModel); //创建表空间 oracleExtract.createTableSpace(conn, collectOracle,
oracleExtract.createUser(conn, collectOracle, oracleModel);//创建用户并授权 oracleModel); // 创建表空间
oracleExtract.extractColleDB(conn, collectOracle, oracleModel);//执行抽取 oracleExtract.createUser(conn, collectOracle,
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "2"); //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成 oracleModel);// 创建用户并授权
cmd = "kubectl label --overwrite rc " oracleExtract.extractColleDB(conn, collectOracle,
+ replicasName + " isExtract=2"; oracleModel);// 执行抽取
// client.updateOrAddReplicasLabelById(collectOracle.getName(),
// "isExtract", "2");
// //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成
cmd = "kubectl label --overwrite rc " + replicasName
+ " isExtract=2";
rList = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); rList = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
sb = new StringBuffer(); sb = new StringBuffer();
for (String string : rList) for (String string : rList)
@ -107,13 +130,16 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
} }
} catch (Exception e) { } catch (Exception e) {
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} } finally {
finally{
// sql日志记录时间 // sql日志记录时间
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper.fileWrite(
+ collectOracle.getName()+".log", "\r\n 汇总结束 \r\n"+ DateForm.date2StringBysecond(new Date()) +"\r\n"); Configs.EXTRACT_LOG_LOCALTION
String cmd = "kubectl label --overwrite rc " + collectOracle.getName() + ".log",
+ replicasName + " isExtract=2"; "\r\n 汇总结束 \r\n"
+ DateForm.date2StringBysecond(new Date())
+ "\r\n");
String cmd = "kubectl label --overwrite rc " + replicasName
+ " isExtract=2";
Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
} }
} }
@ -125,15 +151,21 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
} }
@Override @Override
public boolean isConnectTotalOracle(GatherOracleInfo oracleModel) throws Exception { public boolean isConnectTotalOracle(GatherOracleInfo oracleModel)
throws Exception {
boolean isConnect = false; boolean isConnect = false;
Connection conn = OracleConnector.connectionBuilder("jdbc:oracle:thin:@" + oracleModel.getIp() + ":" + oracleModel.getPort() + ":" Connection conn = OracleConnector.connectionBuilder(
+ oracleModel.getDatabaseName(), oracleModel.getUser(), oracleModel.getPassword(), null); "jdbc:oracle:thin:@" + oracleModel.getIp() + ":"
+ oracleModel.getPort() + ":"
+ oracleModel.getDatabaseName(), oracleModel.getUser(),
oracleModel.getPassword(), null);
if (null == conn) { if (null == conn) {
isConnect = false; isConnect = false;
throw new CustomException(Custom4exception.connect_Oracle_Except, null, oracleModel); throw new CustomException(Custom4exception.connect_Oracle_Except,
null, oracleModel);
// FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION // FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
// + dataInfolist.get(0).getName(), "创建oracle连接失败: [" + conn + "]\r\n"); // + dataInfolist.get(0).getName(), "创建oracle连接失败: [" + conn +
// "]\r\n");
} else { } else {
isConnect = oracleExtract.testConnect(conn); isConnect = oracleExtract.testConnect(conn);
} }
@ -141,7 +173,8 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
} }
@Override @Override
public boolean extractStandardTable(String name, List<OracleConnectorParams> dataInfolist, public boolean extractStandardTable(String name,
List<OracleConnectorParams> dataInfolist,
GatherOracleInfo oracleConnect) throws Exception { GatherOracleInfo oracleConnect) throws Exception {
boolean isSuccess = false; boolean isSuccess = false;
try { try {
@ -152,50 +185,73 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
if (datainfos.size() == 0) { if (datainfos.size() == 0) {
return false; return false;
} }
Connection conn = OracleConnector.connectionBuilder("jdbc:oracle:thin:@" + oracleModel.getIp() + ":" + oracleModel.getPort() + ":" Connection conn = OracleConnector.connectionBuilder(
+ oracleModel.getDatabaseName(), oracleModel.getUser(), oracleModel.getPassword(),dataInfolist.get(0)); "jdbc:oracle:thin:@" + oracleModel.getIp() + ":"
+ oracleModel.getPort() + ":"
+ oracleModel.getDatabaseName(),
oracleModel.getUser(), oracleModel.getPassword(),
dataInfolist.get(0));
if (null == conn) { if (null == conn) {
FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION FileOperateHelper.fileWrite(Configs.EXTRACT_LOG_LOCALTION
+ dataInfolist.get(0).getName()+".log", "创建oracle连接失败: [" + conn + "]\r\n"); + dataInfolist.get(0).getName() + ".log",
"创建oracle连接失败: [" + conn + "]\r\n");
return false; return false;
} }
for (OracleConnectorParams collectOracle : datainfos) { for (OracleConnectorParams collectOracle : datainfos) {
String replicasName = collectOracle.getName(); String replicasName = collectOracle.getName();
try { try {
if(null != collectOracle.getDataId() && !"".equals(collectOracle.getDataId())){ if (null != collectOracle.getDataId()
&& !"".equals(collectOracle.getDataId())) {
DataInfoEntity data = new DataInfoEntity(); DataInfoEntity data = new DataInfoEntity();
data.setId(Integer.valueOf(collectOracle.getDataId())); data.setId(Integer.valueOf(collectOracle.getDataId()));
// 设置为 标准表 抽取中 // 设置为 标准表 抽取中
data.setCheckoutFlag(Constant.CHECKOUTFLAG_SIX); data.setCheckoutFlag(Constant.CHECKOUTFLAG_SIX);
data.setStandardExtractStatus("1"); data.setStandardExtractStatus("1");
dataInfoDao.update(data); dataInfoDao.update(data);
collectOracle.setName("CQ" + collectOracle.getName().replace("-", "_")); collectOracle.setName("CQ"
+ collectOracle.getName().replace("-", "_"));
String cmd = "kubectl annotate --overwrite rc " String cmd = "kubectl annotate --overwrite rc "
+ replicasName + " standardExtractStatus=1"; + replicasName + " standardExtractStatus=1";
// sql日志记录时间 // sql日志记录时间
FileOperateHelper.fileWrite(Configs.EXTRACT_STANDARD_LOG_LOCALTION FileOperateHelper
+ collectOracle.getName()+".log", "\r\n 开始抽取标准表 \r\n"+ DateForm.date2StringBysecond(new Date()) +"\r\n"); .fileWrite(
Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ collectOracle.getName()
+ ".log",
"\r\n 开始抽取标准表 \r\n"
+ DateForm
.date2StringBysecond(new Date())
+ "\r\n");
List<String> rList = Constant.ganymedSSH List<String> rList = Constant.ganymedSSH
.execCmdWaitAcquiescent(cmd); .execCmdWaitAcquiescent(cmd);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
for (String string : rList) for (String string : rList)
sb.append(string).append("\n"); sb.append(string).append("\n");
Configs.CONSOLE_LOGGER.info(sb.toString()); Configs.CONSOLE_LOGGER.info(sb.toString());
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "1"); //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成 // client.updateOrAddReplicasLabelById(collectOracle.getName(),
// "isExtract", "1");
// //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成
oracleExtract.createStandardDBLink(conn, collectOracle); // 创建dblink oracleExtract.createStandardDBLink(conn, collectOracle); // 创建dblink
oracleExtract.createTableSpace(conn, collectOracle, oracleModel); //创建表空间 oracleExtract.createTableSpace(conn, collectOracle,
oracleExtract.createOnlyUser(conn, collectOracle, oracleModel);//创建 抽取标准表的 用户并授权 oracleModel); // 创建表空间
DataInfoEntity tmpdata = dataInfoDao.findById(data.getId()); oracleExtract.createOnlyUser(conn, collectOracle,
oracleModel);// 创建 抽取标准表的 用户并授权
DataInfoEntity tmpdata = dataInfoDao.findById(data
.getId());
if (null != tmpdata) { if (null != tmpdata) {
if (Constant.CHECKOUT_STATUS_THREE.equals(tmpdata.getPayResultLast()) if (Constant.CHECKOUT_STATUS_THREE.equals(tmpdata
|| Constant.CHECKOUT_STATUS_FIVE.equals(tmpdata.getPayResultLast()) .getPayResultLast())
|| Constant.CHECKOUT_STATUS_SIX.equals(tmpdata.getPayResultLast())) { || Constant.CHECKOUT_STATUS_FIVE
.equals(tmpdata.getPayResultLast())
|| Constant.CHECKOUT_STATUS_SIX
.equals(tmpdata.getPayResultLast())) {
// 抽取中 // 抽取中
data.setPayResultLast(Constant.CHECKOUT_STATUS_SIX); data.setPayResultLast(Constant.CHECKOUT_STATUS_SIX);
dataInfoDao.update(data); dataInfoDao.update(data);
boolean isExtrac = true; boolean isExtrac = true;
try { try {
oracleExtract.extractStandardPayTable(conn, collectOracle, oracleModel);//执行抽取 oracleExtract.extractStandardPayTable(conn,
collectOracle, oracleModel);// 执行抽取
} catch (Exception e) { } catch (Exception e) {
// 改回 校验存在的状态 // 改回 校验存在的状态
data.setPayResultLast(Constant.CHECKOUT_STATUS_THREE); data.setPayResultLast(Constant.CHECKOUT_STATUS_THREE);
@ -208,15 +264,19 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
dataInfoDao.update(data); dataInfoDao.update(data);
} }
} }
if (Constant.CHECKOUT_STATUS_THREE.equals(tmpdata.getExecResultLast()) if (Constant.CHECKOUT_STATUS_THREE.equals(tmpdata
|| Constant.CHECKOUT_STATUS_FIVE.equals(tmpdata.getExecResultLast()) .getExecResultLast())
|| Constant.CHECKOUT_STATUS_SIX.equals(tmpdata.getExecResultLast())) { || Constant.CHECKOUT_STATUS_FIVE
.equals(tmpdata.getExecResultLast())
|| Constant.CHECKOUT_STATUS_SIX
.equals(tmpdata.getExecResultLast())) {
// 抽取中 // 抽取中
data.setExecResultLast(Constant.CHECKOUT_STATUS_SIX); data.setExecResultLast(Constant.CHECKOUT_STATUS_SIX);
dataInfoDao.update(data); dataInfoDao.update(data);
boolean isExtrac = true; boolean isExtrac = true;
try { try {
oracleExtract.extractStandardExecTable(conn, collectOracle, oracleModel);//执行抽取 oracleExtract.extractStandardExecTable(
conn, collectOracle, oracleModel);// 执行抽取
} catch (Exception e) { } catch (Exception e) {
// 改回 校验存在的状态 // 改回 校验存在的状态
data.setExecResultLast(Constant.CHECKOUT_STATUS_THREE); data.setExecResultLast(Constant.CHECKOUT_STATUS_THREE);
@ -228,10 +288,13 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
dataInfoDao.update(data); dataInfoDao.update(data);
} }
} }
// client.updateOrAddReplicasLabelById(collectOracle.getName(), "isExtract", "2"); //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成 // client.updateOrAddReplicasLabelById(collectOracle.getName(),
// "isExtract", "2");
// //更新oracle汇总状态0标示为未汇总1标示汇总中2标示汇总完成
cmd = "kubectl annotate --overwrite rc " cmd = "kubectl annotate --overwrite rc "
+ replicasName + " standardExtractStatus=2"; + replicasName + " standardExtractStatus=2";
rList = Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); rList = Constant.ganymedSSH
.execCmdWaitAcquiescent(cmd);
sb = new StringBuffer(); sb = new StringBuffer();
for (String string : rList) for (String string : rList)
sb.append(string).append("\n"); sb.append(string).append("\n");
@ -243,11 +306,14 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
} }
} catch (Exception e) { } catch (Exception e) {
log.error(Custom4exception.OracleSQL_Except, e); log.error(Custom4exception.OracleSQL_Except, e);
} } finally {
finally{
// sql日志记录时间 // sql日志记录时间
FileOperateHelper.fileWrite(Configs.EXTRACT_STANDARD_LOG_LOCALTION FileOperateHelper.fileWrite(
+ collectOracle.getName()+".log", "\r\n 抽取标准表结束 \r\n"+ DateForm.date2StringBysecond(new Date()) +"\r\n"); Configs.EXTRACT_STANDARD_LOG_LOCALTION
+ collectOracle.getName() + ".log",
"\r\n 抽取标准表结束 \r\n"
+ DateForm.date2StringBysecond(new Date())
+ "\r\n");
String cmd = "kubectl annotate --overwrite rc " String cmd = "kubectl annotate --overwrite rc "
+ replicasName + " standardExtractStatus=2"; + replicasName + " standardExtractStatus=2";
Constant.ganymedSSH.execCmdWaitAcquiescent(cmd); Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
@ -260,22 +326,43 @@ public class OracleExtractServiceImpl implements IOracleExtractService {
return isSuccess; return isSuccess;
} }
@Override
public void updateDataExtractStatus(OracleConnectorParams ocp, int status) {
DataInfoEntity data = new DataInfoEntity();
data.setId(Integer.valueOf(ocp.getDataId()));
// 设置为 标准表 抽取中
data.setCheckoutFlag(Constant.CHECKOUTFLAG_SIX);
data.setExtractStatus(status);
try {
dataInfoDao.update(data);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// @Override // @Override
// public boolean extractOracle(String name, List<OracleConnectorParams> dataInfos, GatherOracleInfo oracleConnect) throws Exception { // public boolean extractOracle(String name, List<OracleConnectorParams>
// dataInfos, GatherOracleInfo oracleConnect) throws Exception {
// boolean isSuccess = false; // boolean isSuccess = false;
// try{ // try{
// //map转 bean(汇总库信息-带tableName的) // //map转 bean(汇总库信息-带tableName的)
//// GatherOracleInfo oracleModel = (GatherOracleInfo) Bean2MapUtils.convertMap(GatherOracleInfo.class, oracleConnect); // // GatherOracleInfo oracleModel = (GatherOracleInfo)
// Bean2MapUtils.convertMap(GatherOracleInfo.class, oracleConnect);
// //
// //采集库连接参数 // //采集库连接参数
//// List<OracleConnectorParams> datainfos = new ArrayList<OracleConnectorParams>(); // // List<OracleConnectorParams> datainfos = new
// ArrayList<OracleConnectorParams>();
// // for (Map<String, String> map : dataInfoMap) { // // for (Map<String, String> map : dataInfoMap) {
//// OracleConnectorParams dataInfoEntity = (OracleConnectorParams) Bean2MapUtils.convertMap(OracleConnectorParams.class, oracleConnect); // // OracleConnectorParams dataInfoEntity = (OracleConnectorParams)
// Bean2MapUtils.convertMap(OracleConnectorParams.class, oracleConnect);
// // datainfos.add(dataInfoEntity); // // datainfos.add(dataInfoEntity);
// // } // // }
// //
// Connection conn = OracleConnector.ConnectionBuilder("jdbc:oracle:thin:@" + oracleConnect.getIp() + ":" + oracleConnect.getPort() + "/" // Connection conn = OracleConnector.ConnectionBuilder("jdbc:oracle:thin:@"
// + oracleConnect.getDatabaseName(), oracleConnect.getUser(), oracleConnect.getPassword()); // + oracleConnect.getIp() + ":" + oracleConnect.getPort() + "/"
// + oracleConnect.getDatabaseName(), oracleConnect.getUser(),
// oracleConnect.getPassword());
// //
// for (OracleConnectorParams collectOracle : dataInfos) { // for (OracleConnectorParams collectOracle : dataInfos) {
// //

Loading…
Cancel
Save