diff --git a/src/main/java/net/educoder/ecsonar/config/DynamicDataSourceConfig.java b/src/main/java/net/educoder/ecsonar/config/DynamicDataSourceConfig.java index 92ac9e3..bfd6b45 100644 --- a/src/main/java/net/educoder/ecsonar/config/DynamicDataSourceConfig.java +++ b/src/main/java/net/educoder/ecsonar/config/DynamicDataSourceConfig.java @@ -65,6 +65,10 @@ public class DynamicDataSourceConfig { masterDataSource.setInitialSize(initSize); masterDataSource.setMaxActive(initSize); masterDataSource.setMinIdle(initSize); + masterDataSource.setValidationQuery("select 1"); + masterDataSource.setTestOnBorrow(true); + masterDataSource.setTestWhileIdle(true); + masterDataSource.setTimeBetweenEvictionRunsMillis(60000); return masterDataSource; } diff --git a/src/main/java/net/educoder/ecsonar/services/SonarService.java b/src/main/java/net/educoder/ecsonar/services/SonarService.java index c645b06..63bd703 100644 --- a/src/main/java/net/educoder/ecsonar/services/SonarService.java +++ b/src/main/java/net/educoder/ecsonar/services/SonarService.java @@ -28,6 +28,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -48,13 +50,22 @@ public class SonarService { @Value("${extract.path}") String extractProgramPath; + @Value("${sonar.scan.max-concurrent:1}") + int sonarScanMaxConcurrent; + + @Value("${sonar.scan.retry-times:3}") + int sonarScanRetryTimes; + @Autowired private ExecutorService executorService; private ConcurrentHashMap> concurrentHashMap = new ConcurrentHashMap(8); + private Semaphore sonarScanSemaphore; + @PostConstruct public void init() { + sonarScanSemaphore = new Semaphore(Math.max(1, sonarScanMaxConcurrent)); concurrentHashMap.put(Constant.JAVA, param -> { String command = "sonar-scanner " + "-Dsonar.host.url=" + sonarUrl + " " + @@ -184,7 +195,7 @@ public class SonarService { } - public void sonar(String projectPath, String key) { + public boolean sonar(String projectPath, String key) { String command = "sonar-scanner " + "-Dsonar.host.url=" + sonarUrl + " " + "-Dsonar.sourceEncoding=utf-8 " + @@ -192,8 +203,42 @@ public class SonarService { "-Dsonar.java.binaries=./ " + "-Dsonar.projectBaseDir=" + projectPath; log.info("projectPath:{},key:{}, command: {}", projectPath, key, command); - SystemUtil.ExecuteResp executeResp = SystemUtil.executeAndGetExitStatus(command); - log.info("projectPath:{},key:{}, result: {}", projectPath, key, executeResp.getStatus() != 0 ? executeResp.getOutput() : "success"); + boolean acquired = false; + try { + sonarScanSemaphore.acquire(); + acquired = true; + for (int i = 1; i <= Math.max(1, sonarScanRetryTimes); i++) { + SystemUtil.ExecuteResp executeResp = SystemUtil.executeAndGetExitStatus(command); + if (executeResp.getStatus() == 0) { + log.info("projectPath:{},key:{}, result: success", projectPath, key); + return true; + } + + String output = executeResp.getOutput(); + boolean retryable = isRetryableSonarError(output); + log.warn("projectPath:{},key:{}, sonar scanner failed, attempt:{}/{}, retryable:{}, result:{}", + projectPath, key, i, sonarScanRetryTimes, retryable, output); + if (!retryable || i >= sonarScanRetryTimes) { + return false; + } + TimeUnit.SECONDS.sleep(20L * i); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("projectPath:{},key:{}, sonar scanner interrupted", projectPath, key, e); + } finally { + if (acquired) { + sonarScanSemaphore.release(); + } + } + return false; + } + + private boolean isRetryableSonarError(String output) { + return StringUtils.contains(output, "Error 500") + || StringUtils.contains(output, "/api/rules/search.protobuf") + || StringUtils.contains(output, "Unable to load component class org.sonar.scanner.report.ActiveRulesPublisher") + || StringUtils.contains(output, "Unable to load component interface org.sonar.api.batch.rule.ActiveRules"); } /** @@ -258,7 +303,7 @@ public class SonarService { */ public void scanExcel(String excelPath) { - int homeworkId = 202505142; + int homeworkId = 202605112; try { List personList = ExcelUtil.readExcel(excelPath); diff --git a/src/main/java/net/educoder/ecsonar/task/ReadExcelRunnable.java b/src/main/java/net/educoder/ecsonar/task/ReadExcelRunnable.java index e17d975..c5b9563 100644 --- a/src/main/java/net/educoder/ecsonar/task/ReadExcelRunnable.java +++ b/src/main/java/net/educoder/ecsonar/task/ReadExcelRunnable.java @@ -49,20 +49,24 @@ public class ReadExcelRunnable implements Runnable{ } // for (Person person : sonarRequest.getPersonList()) { // String[] split = StringUtils.split(person.getDownloadUrl(), "/"); -// String zipFilename = UrlUtil.getURLDecoderString(split[split.length - 1]); +//// String zipFilename = UrlUtil.getURLDecoderString(split[split.length - 1]); // try { -// download(person.getDownloadUrl(),path+zipFilename); +// download(person.getDownloadUrl(),path); // } catch (IOException e) { // e.printStackTrace(); // } // } String key = String.format("%d-%s", homeworkId, uid); - if(new File(path + "/.scannerwork/").exists()){ - System.out.println(path+ "----exist"); + File scannerReport = new File(path + "/.scannerwork/report-task.txt"); + if(scannerReport.exists()){ + log.info("{} 已存在成功扫描标记，跳过: {}", key, scannerReport.getAbsolutePath()); return; } log.info("开始调用sonar:{}",key); - sonarService.sonar(path,key); + boolean success = sonarService.sonar(path,key); + if (!success) { + log.error("sonar扫描失败, uid:{}, key:{}, path:{}", uid, key, path); + } } private void download(String zipUrl, String zipPath) throws IOException { @@ -77,8 +81,20 @@ public class ReadExcelRunnable implements Runnable{ String fileName = conn.getHeaderField("Content-Disposition"); if(StringUtils.isNotEmpty(fileName)){ fileName = new String(fileName.getBytes("ISO-8859-1"), "UTF-8"); - fileName = fileName.substring(fileName.indexOf("filename=")+10,fileName.length()-1); + fileName = UrlUtil.getURLDecoderString(fileName); + fileName = UrlUtil.getURLDecoderString(fileName); + + fileName = fileName.substring(fileName.indexOf("filename=")+10); + // 如果存在分号 ;，截取前面部分 + int semicolonIndex = fileName.indexOf(';'); + if (semicolonIndex != -1) { + fileName = fileName.substring(0, semicolonIndex); + } + + // 去掉可能的引号 + fileName = fileName.replaceAll("\"", ""); zipPath = zipPath.substring(0, zipPath.lastIndexOf("/")+1) + fileName; + log.info("zipPath:{}",zipPath); } try (BufferedInputStream in = new BufferedInputStream(conn.getInputStream()); FileOutputStream fileOutputStream = new FileOutputStream(zipPath)) { diff --git a/src/main/java/net/educoder/ecsonar/utils/RarUtil.java b/src/main/java/net/educoder/ecsonar/utils/RarUtil.java index 87201f8..46dd760 100644 --- a/src/main/java/net/educoder/ecsonar/utils/RarUtil.java +++ b/src/main/java/net/educoder/ecsonar/utils/RarUtil.java @@ -34,7 +34,7 @@ public class RarUtil { // un7Z("/tmp/20230301/201905962241/20230519153506_i39sv5p2.7z", "/Users/youyongsheng/Desktop/aa"); - String path = "/tmp/202505142"; + String path = "/tmp/202605112"; // unzip(); // valid(path); // removeFullCode(path); @@ -43,8 +43,8 @@ public class RarUtil { public static void unzip(){ - String sourceDir = "/tmp/202505141"; - String targetDir = "/tmp/202505142"; + String sourceDir = "/tmp/202605111"; + String targetDir = "/tmp/202605112"; File directory = new File(sourceDir); File[] studentDirs = directory.listFiles(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 96f5afd..47f1faf 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -6,7 +6,7 @@ spring.datasource.initSize=20 #spring.datasource.master.url=jdbc:postgresql://127.0.0.1:5432/sonar -spring.datasource.master.url=jdbc:postgresql://106.75.25.158:5432/sonar +spring.datasource.master.url=jdbc:postgresql://106.75.25.158:5432/sonar?connectTimeout=10&socketTimeout=60&tcpKeepAlive=true spring.datasource.master.username=sonar spring.datasource.master.password=sonar spring.datasource.master.driverClassName=org.postgresql.Driver @@ -14,7 +14,7 @@ spring.datasource.master.driverClassName=org.postgresql.Driver #### test ###### spring.datasource.readonly.driverClassName=com.mysql.jdbc.Driver -spring.datasource.readonly.url=jdbc:mysql://testeducoder-public.mysql.polardb.rds.aliyuncs.com:3306/testeducoderweb?userSSL=false&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false +spring.datasource.readonly.url=jdbc:mysql://testeducoder-public.rwlb.rds.aliyuncs.com:3306/testeducoderweb?userSSL=false&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false spring.datasource.readonly.username=testeducoder spring.datasource.readonly.password=TEST@123 diff --git a/src/test/java/net/educoder/ecsonar/EcsonarApplicationTests.java b/src/test/java/net/educoder/ecsonar/EcsonarApplicationTests.java index db7162e..2bc8547 100644 --- a/src/test/java/net/educoder/ecsonar/EcsonarApplicationTests.java +++ b/src/test/java/net/educoder/ecsonar/EcsonarApplicationTests.java @@ -67,8 +67,8 @@ public class EcsonarApplicationTests { public void scanExcel(){ CountDownLatch countDownLatch = new CountDownLatch(1); - String excelPath = "/Users/youyongsheng/Desktop/ZQ/质量分析/2025-05-14/湘潭大学毕业设计检测数据.xlsx"; -// String excelPath = "/Users/youyongsheng/Desktop/湘潭大学计算机科学与工程系2024届.xlsx"; + String excelPath = "/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/毕业设计-湘潭大学网络空间安全专业2026届.xlsx"; +// String excelPath = "/Users/youyongsheng/Desktop/aaaa.xlsx"; sonarService.scanExcel(excelPath); try { @@ -81,13 +81,13 @@ public class EcsonarApplicationTests { @Test public void writeExcel() throws Exception { - int homeworkId = 202505142; - String directory = "/tmp/202505142/"; + int homeworkId = 202605112; + String directory = "/tmp/202605112/"; File file = new File(directory); File[] files = file.listFiles(); System.out.println("学生数：" + files.length); - List personList = ExcelUtil.readExcel("/Users/youyongsheng/Desktop/ZQ/质量分析/2025-05-14/湘潭大学毕业设计检测数据.xlsx"); + List personList = ExcelUtil.readExcel("/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/毕业设计-湘潭大学网络空间安全专业2026届.xlsx"); Map> collect = personList.stream().collect(Collectors.groupingBy(Person::getUid)); @@ -100,7 +100,7 @@ public class EcsonarApplicationTests { Metrics metrics = reportService.getMetrics(projectName); String templatePath = this.getClass().getClassLoader().getResource("template1.xlsx").getPath(); - String outPath = "/Users/youyongsheng/Desktop/bbbb.xlsx"; + String outPath = "/Users/youyongsheng/Desktop/bbbb2.xlsx"; System.out.println("第"+(i++)+"个学生," + uid); if (CollectionUtils.isEmpty(collect.get(uid))) { @@ -110,6 +110,7 @@ public class EcsonarApplicationTests { } + reportService.writeSpecifyTemplateToExcel(uid, collect.get(uid).get(0).getName(), metrics,