diff --git a/.gitignore b/.gitignore index 84adb3f..b98a907 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ # ---> Java # Compiled class file *.class @@ -20,6 +24,33 @@ *.tar.gz *.rar +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5b0f449 --- /dev/null +++ b/pom.xml @@ -0,0 +1,137 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.1.3 + + + com.pc.crawl + vul_repo + 0.0.1-SNAPSHOT + vul_repo + vul_repo + + 8 + + + + org.springframework.boot + spring-boot-starter-web + + + + mysql + mysql-connector-java + 8.0.28 + + + + org.springframework.boot + spring-boot-starter-data-redis + + + redis.clients + jedis + 2.9.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + + com.alibaba + fastjson + 1.2.83 + + + org.springframework.boot + spring-boot-starter-data-jdbc + + + org.springframework.boot + spring-boot-starter + + + com.baomidou + mybatis-plus-boot-starter + 3.5.2 + + + + com.alibaba + druid-spring-boot-starter + 1.1.10 + + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + com.alibaba + easyexcel + 2.1.6 + + + + cn.hutool + hutool-all + 5.7.2 + compile + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + false + true + 1.8 + ${java.version} + ${java.version} + false + UTF8 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + diff --git a/src/main/java/com/pc/crawl/vul_repo/VulRepoApplication.java b/src/main/java/com/pc/crawl/vul_repo/VulRepoApplication.java new file mode 100644 index 0000000..44b49cc --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/VulRepoApplication.java @@ -0,0 +1,15 @@ +package com.pc.crawl.vul_repo; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@MapperScan("com.pc.crawl.vul_repo.mapper") +@SpringBootApplication +public class VulRepoApplication { + + public static void main(String[] args) { + SpringApplication.run(VulRepoApplication.class, args); + } + +} diff --git a/src/main/java/com/pc/crawl/vul_repo/controller/VulnerabilityRepoController.java b/src/main/java/com/pc/crawl/vul_repo/controller/VulnerabilityRepoController.java new file mode 100644 index 0000000..7051b3d --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/controller/VulnerabilityRepoController.java @@ -0,0 +1,59 @@ +package com.pc.crawl.vul_repo.controller; + + +import com.pc.crawl.vul_repo.service.VulnerabilityRepoService; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.*; + +/** + * 漏洞库文件 前端控制器 + * + * @author jshixiong + */ +@RestController +@RequestMapping("/vulnerability/repo") +public class VulnerabilityRepoController { + + @Autowired + private VulnerabilityRepoService vulnerabilityRepoService; + + @GetMapping("/update") + public void updateRepo(){ + new Thread(() -> { + vulnerabilityRepoService.updateRepoFile(); + }).start(); + } + + /** + * 漏洞库文件 + * + */ + @GetMapping("/file") + public void vulnerabilityRepoFile(HttpServletResponse response){ + String content = vulnerabilityRepoService.getRepoFile(); + + response.setCharacterEncoding("utf-8"); + response.setContentType("application/octet-stream"); + response.setHeader("Content-Disposition","attachment;filename=repoFile.json"); + + int len; + byte[] buffer = new byte[1024]; + try (InputStream in = new ByteArrayInputStream(content.getBytes()); + BufferedInputStream bis = new BufferedInputStream(in); + OutputStream out = response.getOutputStream()) { + while ((len = bis.read(buffer)) != -1) { + out.write(buffer, 0, len); + } + out.flush(); + }catch (IOException e){ + e.printStackTrace(); + } + } + +} + diff --git a/src/main/java/com/pc/crawl/vul_repo/dto/VulnerabilityRepoDTO.java b/src/main/java/com/pc/crawl/vul_repo/dto/VulnerabilityRepoDTO.java new file mode 100644 index 0000000..32181d2 --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/dto/VulnerabilityRepoDTO.java @@ -0,0 +1,82 @@ +package com.pc.crawl.vul_repo.dto; + + +import lombok.Data; + +/** + * @author jshixiong + */ +@Data +public class VulnerabilityRepoDTO { + + /** + * 自定义id + */ + private String id; + /** + * 组件厂商 + */ + private String vendor; + + /** + *组件名 + */ + private String product; + + /** + *漏洞影响版本 + */ + private String version; + /** + *组件语言 + */ + private String language; + /** + *漏洞名 + */ + private String name; + /** + *cve编号 + */ + private String cve_id; + /** + *cnnvd编号 + */ + private String cnnvd_id; + /** + *cnvd编号 + */ + private String cnvd_id; + /** + *cwe编号 + */ + private String cwe_id; + /** + *漏洞描述 + */ + private String description; + /** + *漏洞英文描述 + */ + private String description_en; + /** + *漏洞修复建议 + */ + private String suggestion; + /** + *攻击方式 + */ + private String attack_type; + /** + *漏洞发布日期 + */ + private String release_date; + /** + *漏洞风险评级(1~4 风险程度递减) + */ + private Integer security_level_id; + /** + *漏洞利用评级(0:不可利用,1:可利用) + */ + private Integer exploit_level_id; +} diff --git a/src/main/java/com/pc/crawl/vul_repo/entity/VulnerabilityRepo.java b/src/main/java/com/pc/crawl/vul_repo/entity/VulnerabilityRepo.java new file mode 100644 index 0000000..68171f9 --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/entity/VulnerabilityRepo.java @@ -0,0 +1,94 @@ +package com.pc.crawl.vul_repo.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + *

+ * 漏洞库文件 + *

+ * + * @author jshixiong + * @since 2023-03-21 + */ +@Data +@TableName("vulnerability_repo") +public class VulnerabilityRepo { + + @TableId(value = "id",type = IdType.AUTO) + private Integer id; + + /** + * 自定义id + */ + private String xmirrorId; + /** + * 组件厂商 + */ + private String vendor; + + /** + *组件名 + */ + private String product; + + /** + *漏洞影响版本 + */ + private String version; + /** + *组件语言 + */ + private String language; + /** + *漏洞名 + */ + private String name; + /** + *cve编号 + */ + private String cveId; + /** + *cnnvd编号 + */ + private String cnnvdId; + /** + *cnvd编号 + */ + private String cnvdId; + /** + *cwe编号 + */ + private String cweId; + /** + *漏洞描述 + */ + private String description; + /** + *漏洞英文描述 + */ + private String descriptionEn; + /** + *漏洞修复建议 + */ + private String suggestion; + /** + *攻击方式 + */ + private String attackType; + /** + *漏洞发布日期 + */ + private String releaseDate; + /** + *漏洞风险评级(1~4 风险程度递减) + */ + private Integer securityLevelId; + /** + *漏洞利用评级(0:不可利用,1:可利用) + */ + private Integer exploitLevelId; + +} diff --git a/src/main/java/com/pc/crawl/vul_repo/mapper/VulnerabilityRepoMapper.java b/src/main/java/com/pc/crawl/vul_repo/mapper/VulnerabilityRepoMapper.java new file mode 100644 index 0000000..7dff8da --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/mapper/VulnerabilityRepoMapper.java @@ -0,0 +1,19 @@ +package com.pc.crawl.vul_repo.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.pc.crawl.vul_repo.dto.VulnerabilityRepoDTO; +import com.pc.crawl.vul_repo.entity.VulnerabilityRepo; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +/** + * 漏洞库文件 Mapper 接口 + * + * @author jshixiong + */ +@Mapper +public interface VulnerabilityRepoMapper extends BaseMapper { + + List getAll(); +} diff --git a/src/main/java/com/pc/crawl/vul_repo/service/VulnerabilityRepoService.java b/src/main/java/com/pc/crawl/vul_repo/service/VulnerabilityRepoService.java new file mode 100644 index 0000000..05330a1 --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/service/VulnerabilityRepoService.java @@ -0,0 +1,23 @@ +package com.pc.crawl.vul_repo.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.pc.crawl.vul_repo.entity.VulnerabilityRepo; + +/** + * 漏洞库文件 服务类 + * + * @author jshixiong + */ +public interface VulnerabilityRepoService extends IService { + + /** + * 分析仓库漏洞，更新漏洞库文件 + */ + void updateRepoFile(); + + /** + * 获取漏洞库文件 + * @return + */ + String getRepoFile(); +} diff --git a/src/main/java/com/pc/crawl/vul_repo/service/impl/VulnerabilityRepoServiceImpl.java b/src/main/java/com/pc/crawl/vul_repo/service/impl/VulnerabilityRepoServiceImpl.java new file mode 100644 index 0000000..d0f720e --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/service/impl/VulnerabilityRepoServiceImpl.java @@ -0,0 +1,199 @@ +package com.pc.crawl.vul_repo.service.impl; + +import cn.hutool.core.io.FileUtil; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.pc.crawl.vul_repo.dto.VulnerabilityRepoDTO; +import com.pc.crawl.vul_repo.entity.VulnerabilityRepo; +import com.pc.crawl.vul_repo.mapper.VulnerabilityRepoMapper; +import com.pc.crawl.vul_repo.service.VulnerabilityRepoService; +import com.pc.crawl.vul_repo.utils.ShellUtil; +import com.pc.crawl.vul_repo.utils.bean.ShellResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * 漏洞库文件 服务实现类 + * + * @author jshixiong + */ +@Slf4j +@Service +public class VulnerabilityRepoServiceImpl extends ServiceImpl implements VulnerabilityRepoService { + + @Autowired + private VulnerabilityRepoMapper vulnerabilityRepoMapper; + + @Value("${opensca-cli.path}") + private String openScaCliPath; + + @Value("${opensca-cli.pypath:/git/repos/temp/gitee_repo_get2.py}") + private String pyPath; + + @Override + public void updateRepoFile() { +// String pyPath = "/git/repos/temp/gitee_repo_get2.py"; + if (!FileUtil.exist(pyPath)){ + log.error("拉取仓库的python脚本不存在"); + return; + } + //依次爬取每种语言前5页 + String[] array = {"PHP","Kotlin","c","cpp","csharp","Android","Shell","Html","Ruby","Go","Java","NodeJS"}; + List langList = new ArrayList<>(Arrays.asList(array)); + for (String language : langList) { + for (int page = 1;page <= 5;page++){ + String repoPath = "/git/repos/source/git@gitee.com"; + //执行python脚本，爬取仓库 + log.info("language:{}, page:{} 正在爬取仓库...", language, page); + String command = "python " + pyPath + " " + language + " " + page + " " + repoPath; + ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command); + log.info("language:{}, page:{} command:{} 执行python脚本，爬取仓库:{}", language, page, command, shellResult); + + //遍历仓库，挨个解析并更新漏洞库 + log.info("language:{}, page:{} 开始遍历仓库，挨个解析并更新漏洞库...", language, page); + doUpdateRepoFile(repoPath + "/"); + + //清空仓库文件夹repoPath + String rmResult = ShellUtil.execute("rm -rf " + repoPath + "/*"); + log.info("language:{}, page:{} --清空文件夹{}, 结果:{}", language, page, repoPath, rmResult); + } + } + } + + /** + * 遍历仓库，挨个解析并更新漏洞库 + * @param repoPath /git/repos/source/git@gitee.com/ + */ + private void doUpdateRepoFile(String repoPath){ + String command = "ls -l " + repoPath + " |awk '/^d/ {print $NF}'"; + String execute = ShellUtil.execute(command); + if (execute.contains("No such file or directory")){ + log.error("更新漏洞库失败No such file or directory:"+repoPath); + return; + } + + String[] folderList = execute.split("\n"); + for (String folder : folderList){ + folder = repoPath+folder.trim()+"/"; + String command2 = "ls -l " + folder + " |awk '/^d/ {print $NF}'"; + String execute2 = ShellUtil.execute(command2); + if (StringUtils.isEmpty(execute2)){ + log.error("仓库为空:"+folder); + continue; + } + String[] folderList2 = execute2.split("\n"); + for (String folder2 : folderList2){ + folder2 = folder+folder2.trim(); + log.info("开始分析，仓库位置："+folder2); + //分析后将新漏洞加入VulnerabilityRepo + insertRepoByComponent(folder2); + } + } + } + + + /** + * 分析后将新漏洞加入VulnerabilityRepo + * @param fullPath + */ + private void insertRepoByComponent(String fullPath){ + String outPath = fullPath + "/output.json"; + String url = "http://opensca.xmirror.cn:8003"; + String token = "6cb2f6fb-cf87-463d-b5a3-d242eb97d7c1"; + String command = StringUtils.join(openScaCliPath," -url " + , url, " -token ", token, " -path ", fullPath, " -out ", outPath); + ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command); + log.info("insertRepoByComponent command:{}", command); + if (shellResult.getExitStatus() == 0) { + try { + log.info("结果解析"); + String output = FileUtil.readString(outPath, Charset.defaultCharset()); + log.info(output); + JSONObject resultJson = JSONObject.parseObject(output); + JSONArray children = resultJson.getJSONArray("children"); + analyseChildren(children); + }catch (Exception e){ + log.error("结果解析失败，e"+e); + } + + }else { + log.error("组件分析失败 command:{}, result:{}", command, shellResult); + } + } + + /** + * 递归 + * @param children + */ + private void analyseChildren(JSONArray children){ + if (children != null && children.size() > 0) { + for (Object child : children) { + JSONObject json = (JSONObject) child; + if(json.containsKey("vulnerabilities")){ + JSONArray vulnerabilities = json.getJSONArray("vulnerabilities"); + try { + List vulRepos = new ArrayList<>(); + for (int i = 0; i < vulnerabilities.size(); i++) { + JSONObject vul = vulnerabilities.getJSONObject(i); + //原仓库不包含该漏洞才新增 + Wrapper wrapper = Wrappers.lambdaQuery() + .eq(VulnerabilityRepo::getXmirrorId, vul.getString("id")); + if (vulnerabilityRepoMapper.selectList(wrapper).size() == 0) { + VulnerabilityRepo vulRepo = new VulnerabilityRepo(); + vulRepo.setVendor(json.getString("vendor")); + vulRepo.setLanguage(json.getString("language")); + vulRepo.setVersion(json.getString("version")); + vulRepo.setProduct(json.getString("name")); + + vulRepo.setXmirrorId(vul.getString("id")); + vulRepo.setDescription(vul.getString("description")); + vulRepo.setAttackType(vul.getString("attack_type")); + vulRepo.setExploitLevelId(vul.getInteger("exploit_level_id")); + vulRepo.setSecurityLevelId(vul.getInteger("security_level_id")); + vulRepo.setReleaseDate(vul.getString("release_date")); + vulRepo.setSuggestion(vul.getString("suggestion")); + vulRepo.setName(vul.getString("name")); + vulRepo.setCweId(vul.getString("cwe_id")); + vulRepo.setCveId(vul.getString("cve_id")); + vulRepo.setCnvdId(vul.getString("cnvd_id")); + vulRepo.setCnnvdId(vul.getString("cnnvd_id")); + vulRepos.add(vulRepo); + } + } + if (vulRepos.size()>0){ + saveBatch(vulRepos); + } + }catch (Exception e){ + log.error("VulnerabilityRepo新增失败，vulnerabilities："+vulnerabilities); + log.error("VulnerabilityRepo,exception:"+e); + } + } + JSONArray childrenRoot = json.getJSONArray("children"); + if (childrenRoot != null) { + analyseChildren(childrenRoot); + } + } + }else { + log.info("children为空"); + } + } + + @Override + public String getRepoFile() { + List vulnerabilityRepos = vulnerabilityRepoMapper.getAll(); + return JSON.toJSONString(vulnerabilityRepos, SerializerFeature.WriteMapNullValue); + } +} diff --git a/src/main/java/com/pc/crawl/vul_repo/utils/ShellUtil.java b/src/main/java/com/pc/crawl/vul_repo/utils/ShellUtil.java new file mode 100644 index 0000000..ea57e1a --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/utils/ShellUtil.java @@ -0,0 +1,178 @@ +package com.pc.crawl.vul_repo.utils; + +import com.pc.crawl.vul_repo.utils.bean.ShellResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** + * @author guange + * @date 26/02/2017 + */ +public final class ShellUtil { + + private static final Logger logger = LoggerFactory.getLogger(ShellUtil.class); + + /** + * 执行shell命令并获取输出 + */ + public static String execute(String command) { + return executeAndGetExitStatus(command).getOut(); + } + + /** + * 执行shell命令并获得输出及退出码，失败重试 共尝试retryTimes次 + */ + public static ShellResult executeAndGetExitStatus(String command, int retryTimes) { + ShellResult result = new ShellResult(); + + for (int i = 0; i < retryTimes; i++) { + result = executeAndGetExitStatus(command); + if (result.getExitStatus() != 0) { + logger.info("执行shell错误, 再次执行 command: {}, result: {}, times: {}", command, result, i); + } else { + break; + } + } + + return result; + } + + /** + * 执行命令并获得输出以及退出码 + */ + public static ShellResult executeAndGetExitStatus(String command) { + ShellResult result = new ShellResult(); + + StringBuilder out = new StringBuilder(); + Integer exitStatus = -1; + + ProcessBuilder pb = new ProcessBuilder("/bin/sh", "-c", command); + pb.redirectErrorStream(true); + try { + Process process = pb.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + out.append(line); + out.append(System.getProperty("line.separator")); + } + exitStatus = process.waitFor(); + + } catch (Exception e) { + logger.error("执行shell出错, command:{}", command, e); + } + + result.setOut(out.toString().trim()); + result.setExitStatus(exitStatus); + logger.debug("execute shell command: {}, out: {}, status: {}", command, out, exitStatus); + + return result; + } + +// /** +// * 执行命令并获得输出以及退出码 +// */ +// public static ShellResult executeAndGetExitStatus(String command, ShellExeCallBack callBack, T param) { +// ShellResult result = new ShellResult(); +// +// StringBuilder out = new StringBuilder(); +// Integer exitStatus = -1; +// +// ProcessBuilder pb = new ProcessBuilder("/bin/sh", "-c", command); +// pb.redirectErrorStream(true); +// try { +// Process process = pb.start(); +// BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); +// String line; +// while ((line = reader.readLine()) != null) { +// // 分布输出信息固定为一行,若需要处理多行信息从历史commit"评测分布输出多行处理版本提交"找回 +// boolean processed = callBack.processLine(line, param); +// if (processed) { +// continue; +// } +// +// out.append(line); +// out.append(System.getProperty("line.separator")); +// } +// exitStatus = process.waitFor(); +// +// } catch (Exception e) { +// logger.error("执行shell出错, command:{}", command, e); +// } +// +// result.setOut(out.toString().trim()); +// result.setExitStatus(exitStatus); +// +// return result; +// } +// +// +// /** +// * 执行shell命令 +// * @param cmd +// * @param callBack +// * @param param +// * @param +// * @return +// */ +// public static ShellResult executeAndGetErrorStatus(String cmd,ShellExeCallBack callBack, T param) { +// ShellResult result = new ShellResult(); +// StringBuffer sb = new StringBuffer(); +// Integer exitStatus = -1; +// try { +// Process ps = Runtime.getRuntime().exec(cmd); +// +// //获取执行正确结果 +// BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream())); +// String line; +// while ((line = br.readLine()) != null) { +// // 分布输出信息固定为一行,若需要处理多行信息从历史commit"评测分布输出多行处理版本提交"找回 +// boolean processed = callBack.processLine(line, param); +// if (processed) { +// continue; +// } +// +// sb.append(line).append(System.getProperty("line.separator")); +// } +// exitStatus = ps.waitFor(); +// }catch (Exception e) { +// logger.error("执行shell出错, command:{}", cmd, e); +// } +// result.setOut(sb.toString().trim()); +// result.setExitStatus(exitStatus); +// return result; +// } + + /** + * 执行shell命令 + * @param cmd + * @param + * @return + */ + public static ShellResult executeAndGetErrorStatus(String cmd) { + ShellResult result = new ShellResult(); + StringBuffer sb = new StringBuffer(); + Integer exitStatus = -1; + try { + Process ps = Runtime.getRuntime().exec(cmd); + + //获取执行正确结果 + BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream())); + String line; + while ((line = br.readLine()) != null) { + sb.append(line).append(System.getProperty("line.separator")); + } + exitStatus = ps.waitFor(); + }catch (Exception e) { + logger.error("执行shell出错, command:{}", cmd, e); + } + result.setOut(sb.toString().trim()); + result.setExitStatus(exitStatus); + return result; + } + + +} diff --git a/src/main/java/com/pc/crawl/vul_repo/utils/bean/ShellResult.java b/src/main/java/com/pc/crawl/vul_repo/utils/bean/ShellResult.java new file mode 100644 index 0000000..03948d4 --- /dev/null +++ b/src/main/java/com/pc/crawl/vul_repo/utils/bean/ShellResult.java @@ -0,0 +1,48 @@ +package com.pc.crawl.vul_repo.utils.bean; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * shell执行结果 + * + * @author 威少 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ShellResult { + /** + * 退出码 + */ + private Integer exitStatus; + /** + * 实际输出 + */ + private String out; + + public enum ExitStatus { + /** + * 成功 + */ + SUCCESS(0), + /** + * 超时 + */ + TIMEOUT(124), + /** + * 默认失败 + */ + FAIL(-1); + + private int code; + ExitStatus(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..b4f2407 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,53 @@ +server: + port: 8765 + servlet: + context-path: /crawl +spring: + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://rm-bp13v5020p7828r5rso.mysql.rds.aliyuncs.com:3306/quality_analysis?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false + username: testeducoder + password: TEST@123 + type: com.alibaba.druid.pool.DruidDataSource + druid: + initial-size: 20 + max-active: 40 + min-idle: 20 + validation-query: select 1 + test-on-borrow: false + test-on-return: false + test-while-idle: true + data: + redis: + host: 127.0.0.1 + port: 6379 + password: + database: 0 + jedis: + pool: + max-active: 8 + max-wait: -1 + max-idle: 8 + min-idle: 8 + + +mybatis-plus: + #实体类所在包，允许用实体类类名作为别名 + type-aliases-package: com.pc.crawl.vul_repo.entity + # xml扫描，多个目录用逗号或者分号分隔(告诉 Mapper 所对应的 XML 文件位置) + mapper-locations: classpath:mapper/*.xml + configuration: + #配置日志 + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + # 是否开启自动驼峰命名规则映射:从数据库列名到Java属性驼峰命名的类似映射 + map-underscore-to-camel-case: true + # 如果查询结果中包含空值的列，则 MyBatis 在映射的时候，不会映射这个字段 + # 允许在resultType="map"时映射null值 + call-setters-on-nulls: true + + +logging: + level: + root: info + file: + name: all.log \ No newline at end of file diff --git a/src/main/resources/mapper/VulnerabilityRepoMapper.xml b/src/main/resources/mapper/VulnerabilityRepoMapper.xml new file mode 100644 index 0000000..5e42383 --- /dev/null +++ b/src/main/resources/mapper/VulnerabilityRepoMapper.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id, xmirror_id, vendor, product, version, language, name, cve_id, cnnvd_id, cnvd_id, cwe_id, description, description_en, suggestion, attack_type, release_date, security_level_id, exploit_level_id, create_time, update_time + + + + diff --git a/src/test/java/com/pc/crawl/vul_repo/VulRepoApplicationTests.java b/src/test/java/com/pc/crawl/vul_repo/VulRepoApplicationTests.java new file mode 100644 index 0000000..0078b47 --- /dev/null +++ b/src/test/java/com/pc/crawl/vul_repo/VulRepoApplicationTests.java @@ -0,0 +1,13 @@ +package com.pc.crawl.vul_repo; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class VulRepoApplicationTests { + + @Test + void contextLoads() { + } + +}