add md5 and getfoldersize

glusterfs-api
lily 9 years ago
parent d3498ac95e
commit 6ce9f75ee7

@ -62,7 +62,8 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN"
</sql>
<!-- 获取数据符合筛选条件的全部记录信息 -->
<select id="getLimitedDataInfoEntities" parameterType="PagerOptions"
<select id="3
" parameterType="PagerOptions"
resultMap="getEntityByText">
SELECT
id,regionalism_code,city_name,district_name,system_code,system_name,data_type,data_version,submitted_batch,data_path,data_charset,collection_time,collector_name,collector_contacts,data_year

@ -0,0 +1,106 @@
package com.platform.glusterfs;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.aspectj.weaver.ast.And;
import com.mysql.jdbc.log.Log;
import com.platform.utils.Constant;
public class CheckoutMD5 {
public static Logger log = Logger.getLogger(CheckoutMD5.class);
String sourcePath;
String destPath;
String dataName;
// String cmd_crateSourceMD5File="find "+sourcePath+dataName+"/app/ -type f
// -print0 | xargs -0 md5sum | sort >"+deskPath+dataName+"_md5.txt";
String cmd_getSourceMD5File;
// String cmd_crateDestMD5File="find "+destPath+dataName+"/app/ -type f
// -print0 | xargs -0 md5sum | sort >"+deskPath+dataName+"_md5.txt";
String cmd_getDestMD5File;
Map<String, String> source_md5 = new HashMap<String, String>();
Map<String, String> dest_md5 = new HashMap<String, String>();
public CheckoutMD5() {
// TODO Auto-generated constructor stub
}
public CheckoutMD5(String sourcePath, String destPath, String dataName) {
// TODO Auto-generated constructor stub
this.sourcePath = sourcePath;
this.destPath = destPath;
this.dataName = dataName;
cmd_getSourceMD5File = "find " + sourcePath + dataName + "/app/ -type f -print0 | xargs -0 md5sum | sort ";
cmd_getDestMD5File = "find " + destPath + dataName + "/app/ -type f -print0 | xargs -0 md5sum | sort ";
}
/**
* sourcePathdestPath1
* 0MD5-1-2-3
*
* @param sourcePath
* @param destPath
* @return
* @see [##]
*/
public int checkoutMD5Folder(String sourcePath, String destPath) {
log.info("start checkout md5 "+sourcePath+" and "+ destPath);
List<String> wrong_files = new ArrayList<String>();
String source_cmd = "find " + sourcePath + " -type f -print0 | xargs -0 md5sum";
String dest_cmd = "find " + destPath + " -type f -print0 | xargs -0 md5sum";
List<String> sourceReStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(source_cmd);
if (sourceReStrings == null || sourceReStrings.size() == 0) {
log.error("get " + sourcePath + " MD5 error!");
return -1;
}
if(sourceReStrings.get(0).contains(Constant.noSuchFile)){
log.error(sourcePath+" is not exist!");
return -2;
}
List<String> destReStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(dest_cmd);
if (destReStrings == null || destReStrings.size() == 0) {
log.error("get " + destReStrings + " MD5 error!");
return -1;
}
if(destReStrings.get(0).contains(Constant.noSuchFile)){
log.error(destPath+" is not exist!");
return -3;
}
Map<String, String> source_md5 = new HashMap<String, String>();
Map<String, String> dest_md5 = new HashMap<String, String>();
for (String line : sourceReStrings) {
String[] lines = line.split(" ");
String key = lines[1].replace(sourcePath, "").trim();
String value = lines[0].trim();
source_md5.put(key, value);
}
for (String line : destReStrings) {
String[] lines = line.split(" ");
String key = lines[1].replace(destPath, "").trim();
String value = lines[0].trim();
dest_md5.put(key, value);
}
for (Map.Entry<String, String> mapEntry : source_md5.entrySet()) {
if (!(dest_md5.containsKey(mapEntry.getKey())
&& dest_md5.get(mapEntry.getKey()).equals(mapEntry.getValue()))) {
log.info(sourcePath + " and " + destPath + " is not same!");
return 0;
// System.out.println(mapEntry.getKey());
}
}
log.info(sourcePath + " and " + destPath + " is same!");
return 1;
}
public static void main(String[] args) {
CheckoutMD5 checkoutMD5 = new CheckoutMD5();
System.out.println(checkoutMD5.checkoutMD5Folder("/home/v1_copy","/home/ubuntu"));
}
}

@ -31,7 +31,7 @@ public class ClusterInfo {
log.info("get cluster info");
Map<String, String> peerIps = new HashMap<String, String>();
List<String> reStrings = Constant.ganymedSSH.execCmdWait(Constant.hostIp, Constant.rootUser, Constant.rootPasswd, Constant.port, Constant.glusterPeerStatus);
List<String> reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(Constant.glusterPeerStatus);
if (reStrings == null) {
log.error("1101 command get result is null");
return null;
@ -104,7 +104,7 @@ public class ClusterInfo {
public static void main(String[] args) {
PropertyConfigurator.configure("log4j.properties");
// PropertyConfigurator.configure("log4j.properties");
System.out.println(new ClusterInfo().showClusterInfo());
}
}

@ -68,16 +68,17 @@ public class GetTreeData {
*/
public FolderNode getDatasWithShell(String name) {
if(name.endsWith("/")){
name=name.substring(0, name.length());
name=name.substring(0, name.length()-1);
}
String names[]=name.split("/");
String only_name=names[names.length-1];
// String names[]=name.split("/");
// String only_name=names[names.length-1];
FolderNode fileNode = new FolderNode(name);
fileNode.setPath(name);
String shellComment=new FileOperateHelper().fileReaderAndendline(Constant.fileGetTreeData);
// String shellComment= new FileOperateHelper().fileReaderAndendline(Constant.fileGetTreeData);
String shellComment= Constant.strGetTreeData;
String sh_path="/getTreedata.sh";
String cmd="echo -e "+shellComment+" > "+sh_path+" & chmod +x "+sh_path;
String cmd="echo -e \""+shellComment+"\" > "+sh_path+" & chmod +x "+sh_path;
Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
// Map<String, String> files = showData.showFolderData(name);
List<String> files=Constant.ganymedSSH.execCmdWaitAcquiescent(sh_path+" "+name);
@ -97,7 +98,8 @@ public class GetTreeData {
public void test_getTreeData() {
GetTreeData getTreeData=new GetTreeData();
FolderNode fileOrFolder=getTreeData.getDatas("/home");
// FolderNode fileOrFolder=getTreeData.getDatas("/home/gfs_ftp_point");
FolderNode fileOrFolder=getTreeData.getDatasWithShell("/home/gfs_ftp_point/");
System.out.println(fileOrFolder);
}
}

@ -1,11 +1,65 @@
package com.platform.glusterfs;
import java.util.List;
import org.apache.log4j.Logger;
import org.hamcrest.core.Is;
import com.platform.utils.Constant;
import com.platform.utils.Support;
public class SetCluster {
public static Logger log = Logger.getLogger ( SetCluster.class);
public int addPeer(String peerip){
/**
*
* 1-1ip-2 0
* @param peerip
* @return
* @see [##]
*/
public int probePeer(String peerip){
if(!Support.checkIP(peerip)){
log.error(peerip +"is illegal!" );
return -1;
}
String cmd="gluster peer probe "+peerip;
List<String> reStrings=Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
if(reStrings==null || reStrings.size()==0){
log.error("detach error! ");
return -2;
}
if(reStrings.contains(Constant.success)){
log.info("probe success!");
return 1;
}
log.info("probe failed!");
return 0;
}
/**
*
* 1-1ip-2 0
* @param peerip
* @return
* @see [##]
*/
public int detachPeer(String peerip){
if(!Support.checkIP(peerip)){
log.error(peerip +"is illegal!" );
return -1;
}
String cmd="gluster peer detach "+peerip;
List<String> reStrings=Constant.ganymedSSH.execCmdWaitAcquiescent(cmd);
if(reStrings==null || reStrings.size()==0){
log.error("detach error! ");
return -2;
}
if(reStrings.contains(Constant.success)){
log.info("detach success!");
return 1;
}
log.info("detach failed!");
return 0;
}
}

@ -7,99 +7,126 @@ import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.junit.Test;
import com.platform.utils.Constant;
public class ShowData {
public static Logger log = Logger.getLogger ( ShowData.class);
public static Logger log = Logger.getLogger(ShowData.class);
/**
* get the data of volumeName Map<string s1,string s2> s1 is data name and s2 is type file or folder
* <>
* get the data of volumeName Map<string s1,string s2> s1 is data name and
* s2 is type file or folder <>
*
* @param volumeName
* @return
* @see [##]
*/
public Map<String,String> showVolumeFiles(String volumeName){
log.info("start show the data");
Map<String,String> data_type=new HashMap<String, String>();
public Map<String, String> showVolumeFiles(String volumeName) {
log.info("start show the data");
Map<String, String> data_type = new HashMap<String, String>();
/**
* get mount point of volumeName
*/
String folderName=volumeName;
data_type=showFolderData(volumeName);
return data_type;
}
/**
* get the data of folder name
* Map<String,String> is folder name and type 1 is file and others is folder
String folderName = volumeName;
data_type = showFolderData(volumeName);
return data_type;
}
/**
* get the data of folder name Map<String,String> is folder name and type 1
* is file and others is folder
*
*
* @param FolderName
* @return
*/
public Map<String,String> showFolderData(String folderName){
log.info(" start get "+folderName+" data");
public Map<String, String> showFolderData(String folderName) {
log.info(" start get " + folderName + " data");
Map<String, String> data_type = new HashMap<String, String>();
String command = "ls -l " + folderName;
Map<String,String> data_type=new HashMap<String, String>();
String command="ls -l "+folderName;
/*
RunCommand runCommand=new RunCommand();
List<String> reStrings=runCommand.runCommandWait(command);
*/
List<String> reStrings = Constant.ganymedSSH.execCmdWait(Constant.hostIp, Constant.rootUser, Constant.rootPasswd, Constant.port, command);
if(reStrings==null){
* RunCommand runCommand=new RunCommand(); List<String>
* reStrings=runCommand.runCommandWait(command);
*/
List<String> reStrings = Constant.ganymedSSH.execCmdWait(Constant.hostIp, Constant.rootUser,
Constant.rootPasswd, Constant.port, command);
if (reStrings == null) {
log.error("2101 command get result is null");
return null;
}
if(reStrings.size()==0){
if (reStrings.size() == 0) {
log.info("2102 the folder is empty");
return data_type;
}
if(reStrings.get(0).contains("No such file or directory")){
log.info("2103 the "+folderName+" is not exists");
if (reStrings.get(0).contains("No such file or directory")) {
log.info("2103 the " + folderName + " is not exists");
return null;
}
/**
* remove first line total number
*/
reStrings.remove(0);
for(Iterator it2 = reStrings.iterator();it2.hasNext();){
String line=(String)it2.next();
line=line.replaceAll(" +", " ");
String keyValue[]=line.split(" ");
if(keyValue.length<9){
log.error("2104 "+line+" length is short");
for (Iterator it2 = reStrings.iterator(); it2.hasNext();) {
String line = (String) it2.next();
line = line.replaceAll(" +", " ");
String keyValue[] = line.split(" ");
if (keyValue.length < 9) {
log.error("2104 " + line + " length is short");
continue;
}
data_type.put(keyValue[8], keyValue[1]);
}
log.info(" get "+folderName+" data successed");
log.info(" get " + folderName + " data successed");
return data_type;
}
/**
*
* <>
* <>
* folder
* -2-1folderfolder
* @param folderPath
* @return
* @see [##]
*/
@Test
public void testShowData(){
public long getFolderSize(String folderPath) {
log.info("get " + folderPath + " Size ");
String command = "du -k -d 0 "+folderPath+" | grep " + folderPath + "|awk \'{print $1}\'";
List<String> reStrings = Constant.ganymedSSH.execCmdWaitAcquiescent(command);
if(reStrings==null || reStrings.size()==0){
log.error("get " + folderPath + " Size error!");
return -2;
}
if (reStrings.get(0).contains(Constant.noSuchFile)) {
log.error(folderPath+" is not exists");
return -1;
}
long size = Long.valueOf(reStrings.get(0));
return size;
}
//@Test
public void testShowData() {
System.out.println(showFolderData("/home"));
}
@Test
public void test_getFolderSize() {
System.out.println(getFolderSize("/home/ubuntu"));
}
}

@ -210,7 +210,7 @@ public class VolumeInfo {
public List<String> getVolumeMountPoint(String volumeName) {
log.info("get volume MountPoint");
List<String> mountPoints = new ArrayList<>();
List<String> reStrings = Constant.ganymedSSH.execCmdWait(Constant.hostIp, Constant.rootUser, Constant.rootPasswd,
Constant.port, Constant.df + "|grep " + volumeName + "|awk '{print $6}'");
// System.out.println(reStrings);
@ -220,7 +220,7 @@ public class VolumeInfo {
}
if(reStrings.size()==0){
log.error("11002 "+volumeName+" is no mountpoint");
return null;
return mountPoints;
}
char flag = reStrings.get(0).trim().toCharArray()[0];
@ -228,7 +228,7 @@ public class VolumeInfo {
log.error("11003 get result string wrong");
return null;
}
List<String> mountPoints = new ArrayList<>();
for(String mountPoint:reStrings){
mountPoint=mountPoint.replaceAll(" ", "");
mountPoints.add(mountPoint);
@ -302,12 +302,12 @@ public class VolumeInfo {
return brick_size;
}
@Test
//@Test
public void test_getVolumebricksDataSize() {
System.out.println(getVolumebricksDataSize("gfs_ftp"));
}
@Test
//@Test
public void test_getVolumebricksAvailableSize() {
System.out.println(getVolumebricksAvailableSize("gfs_ftp"));
}
@ -316,4 +316,10 @@ public class VolumeInfo {
public void test_getVolumeBricks() {
getVolumeBricks("gfs_ftp");
}
@Test
public void test_getVolumeMountPoint() {
System.out.println(getVolumeMountPoint("gfs_ftp"));
System.out.println(getVolumeMountPoint("v1"));
}
}

@ -1,3 +1,4 @@
package com.platform.service;
import org.springframework.ui.ModelMap;

@ -6,17 +6,25 @@ public class Constant {
public static String rootUser = "root";
public static String rootPasswd = "root";
public static String hostIp = "192.168.0.116";
// public static String hostIp = "192.168.1.105";
// public static String hostIp = "192.168.1.105";
public static int port = 22;
public static String glusterPeerStatus = "gluster peer status";
public static String glusterPeerStatus = "gluster peer status";
public static String glusterVolumeInfo = "gluster volume info ";
public static String df = "df -k ";
public static String peerincluster_connected="PeerinCluster(Connected)";
public static String peerincluster_disconnected="PeerinCluster(Disconnected)";
public static String distributed="distributed";
public static String replica="replica";
public static String stripe="stripe";
public static String noVolume="No volumes present";
public static GanymedSSH ganymedSSH=new GanymedSSH(hostIp, rootUser, rootPasswd, port);
public static String fileGetTreeData="./WEB-INF/config/getTreedata.sh";
public static String peerincluster_connected = "PeerinCluster(Connected)";
public static String peerincluster_disconnected = "PeerinCluster(Disconnected)";
public static String distributed = "distributed";
public static String replica = "replica";
public static String stripe = "stripe";
public static String noVolume = "No volumes present";
public static String success = "success";
public static String failed = "failed";
public static String noSuchFile = "No such file or directory";
public static GanymedSSH ganymedSSH = new GanymedSSH(hostIp, rootUser, rootPasswd, port);
public static String fileGetTreeData = "WebContent\\WEB-INF\\config\\getTreedata.sh";
public static String strGetTreeData = "function ergodic(){\n "
+ "for file in \\`ls \\$1\\`\n do\n if [ \"\\$file\" != \"app\" -a -d \\$1\\\"/\\\"\\$file ]\n "
+ "then\n ergodic \\$1\"/\"\\$file\n else\n local path=\\$1\"/\"\\$file\n "
+ "echo \\$path \n fi\n done\n}\n\nIFS=\\$\\'\\n\\' "
+ "#这个必须要,否则会在文件名中有空格时出错\nINIT_PATH=\".\";\nergodic \\$1\n";
}

@ -7,6 +7,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import org.junit.Test;
/**
*
*
@ -65,14 +67,14 @@ public class FileOperateHelper {
* @param path
* @return
*/
@SuppressWarnings("resource")
public static String fileReaderAndendline(String path) {
public String fileReaderAndendline(String path) {
StringBuffer sb = new StringBuffer();
String tempString = "";
try {
File file = new File(path);
if (!file.exists())
return "";
return "";
FileInputStream fis = new FileInputStream(file);
BufferedReader br = new BufferedReader(new InputStreamReader(fis));
while ((tempString = br.readLine()) != null) {
@ -83,4 +85,10 @@ public class FileOperateHelper {
}
return sb.toString();
}
@Test
public void test_fileReaderAndendline(){
System.out.println(System.getProperty("user.dir"));
System.out.println(fileReaderAndendline(Constant.fileGetTreeData));
}
}

@ -111,7 +111,7 @@ public class GanymedSSH {
BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout));
BufferedReader stderrReader = new BufferedReader(new InputStreamReader(stderr));
while (true) {
String line = stdoutReader.readLine();
String line = stderrReader.readLine();
if (line != null) {
// System.out.println(line);
@ -120,18 +120,19 @@ public class GanymedSSH {
break;
}
}
if(reStrings.size()==0){
while (true) {
String line = stderrReader.readLine();
while (true) {
String line = stdoutReader.readLine();
if (line != null) {
// System.out.println(line);
reStrings.add(line);
} else {
break;
if (line != null) {
// System.out.println(line);
reStrings.add(line);
} else {
break;
}
}
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@ -188,8 +189,7 @@ public class GanymedSSH {
return reStrings;
}
public Map<String, String> execMD5cmd(String host, String username, String password, int port, String cmd,
String prefix) {
public Map<String, String> execMD5cmd(String cmd) {
Map<String, String> md5 = new HashMap<String, String>();
@ -199,22 +199,22 @@ public class GanymedSSH {
sess = conn.openSession();
// 执锟斤拷cmd
sess.execCommand(cmd);
InputStream stdout = new StreamGobbler(sess.getStdout());
@SuppressWarnings("resource")
BufferedReader br = new BufferedReader(new InputStreamReader(stdout));
InputStream stderr = new StreamGobbler(sess.getStderr());
BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout));
BufferedReader stderrReader = new BufferedReader(new InputStreamReader(stderr));
while (true) {
String line = br.readLine();
String line = stdoutReader.readLine();
if (line != null) {
String[] lines = line.split(" ");
String key = lines[1].replace(prefix, "");
String value = lines[0];
String key = lines[1].trim();
String value = lines[0].trim();
md5.put(key, value);
// System.out.println(key+"\t"+value);
} else {
break;
}
}
} catch (IOException e) {
// TODO Auto-generated catch block

@ -0,0 +1,31 @@
/**
* : Support.java
* : </>
* : <>
* @author liliy
* : <>
* 2016912
* <>
*/
package com.platform.utils;
import java.util.regex.Pattern;
/**
* <>
* <>
* @author liliy
* @version [2016912]
* @see [/]
* @since [/]
*/
public class Support {
public static boolean checkIP(String str) {
Pattern pattern = Pattern
.compile("^((\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5]"
+ "|[*])\\.){3}(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5]|[*])$");
return pattern.matcher(str).matches();
}
}

@ -1,14 +0,0 @@
package com.platform.utils;
import java.util.ArrayList;
import java.util.List;
public class TestSupport {
public List<String> strToList(String str) {
List<String> reStrings=new ArrayList<String>();
for(String one:str.split("\n")){
reStrings.add(one);
}
return reStrings;
}
}
Loading…
Cancel
Save