202605 2026届湘潭大学计算机科学与技术系质量分析

manual-analysis
youys 2 weeks ago
parent 23368fbab8
commit ec35d45772

@ -50,12 +50,15 @@ public class SonarService {
@Value("${extract.path}")
String extractProgramPath;
@Value("${sonar.scan.max-concurrent:1}")
@Value("${sonar.scan.max-concurrent:2}")
int sonarScanMaxConcurrent;
@Value("${sonar.scan.retry-times:3}")
int sonarScanRetryTimes;
@Value("${sonar.scan.timeout-minutes:3}")
int sonarScanTimeoutMinutes;
@Autowired
private ExecutorService executorService;
@ -208,7 +211,11 @@ public class SonarService {
sonarScanSemaphore.acquire();
acquired = true;
for (int i = 1; i <= Math.max(1, sonarScanRetryTimes); i++) {
SystemUtil.ExecuteResp executeResp = SystemUtil.executeAndGetExitStatus(command);
SystemUtil.ExecuteResp executeResp = SystemUtil.executeAndGetExitStatus(
command,
Math.max(1, sonarScanTimeoutMinutes),
TimeUnit.MINUTES,
line -> log.info("sonar-scanner key:{} | {}", key, line));
if (executeResp.getStatus() == 0) {
log.info("projectPath:{},key:{}, result: success", projectPath, key);
return true;
@ -295,6 +302,78 @@ public class SonarService {
}
public void downloadExcelArchives(String excelPath, int homeworkId) {
String basePath = String.format("/tmp/%d/", homeworkId);
try {
List<Person> personList = ExcelUtil.readExcel(excelPath);
Map<String, List<Person>> collect = personList.stream().collect(Collectors.groupingBy(Person::getUid));
collect.forEach((uid, persons) -> {
String studentPath = basePath + uid + "/";
File studentDir = new File(studentPath);
if (!studentDir.exists()) {
studentDir.mkdirs();
}
for (Person person : persons) {
try {
downloadToDirectory(person.getDownloadUrl(), studentPath);
} catch (Exception e) {
log.error("下载学生压缩包失败, homeworkId:{}, uid:{}, url:{}",
homeworkId, uid, person.getDownloadUrl(), e);
}
}
});
} catch (Exception e) {
log.error("读取Excel下载压缩包失败: {}", excelPath, e);
}
}
private void downloadToDirectory(String zipUrl, String directory) throws IOException {
log.info("下载文件到目录: {} ---> {}", zipUrl, directory);
String cookie = "_educoder_session=0bea28f7a6db39cefb2fc3cbe715d840;";
URL url = new URL(zipUrl);
URLConnection conn = url.openConnection();
conn.setRequestProperty("Cookie", cookie);
conn.setDoInput(true);
String fileName = getDownloadFileName(zipUrl, conn);
String savePath = directory + fileName;
File saveFile = new File(savePath);
if (saveFile.exists() && saveFile.length() > 0) {
log.info("文件已存在,跳过下载: {}", savePath);
return;
}
try (BufferedInputStream in = new BufferedInputStream(conn.getInputStream());
FileOutputStream fileOutputStream = new FileOutputStream(savePath)) {
byte dataBuffer[] = new byte[1024];
int bytesRead;
while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
fileOutputStream.write(dataBuffer, 0, bytesRead);
}
}
log.info("下载完成: {}", savePath);
}
private String getDownloadFileName(String zipUrl, URLConnection conn) throws UnsupportedEncodingException {
String fileName = conn.getHeaderField("Content-Disposition");
if(StringUtils.isNotEmpty(fileName)){
fileName = new String(fileName.getBytes("ISO-8859-1"), "UTF-8");
fileName = UrlUtil.getURLDecoderString(fileName);
fileName = UrlUtil.getURLDecoderString(fileName);
fileName = fileName.substring(fileName.indexOf("filename=")+10);
int semicolonIndex = fileName.indexOf(';');
if (semicolonIndex != -1) {
fileName = fileName.substring(0, semicolonIndex);
}
return fileName.replaceAll("\"", "");
}
String[] split = StringUtils.split(zipUrl, "/");
return UrlUtil.getURLDecoderString(split[split.length - 1]);
}
/**
* excel
@ -304,7 +383,10 @@ public class SonarService {
public void scanExcel(String excelPath) {
int homeworkId = 202605112;
scanExcel(excelPath, homeworkId);
}
public void scanExcel(String excelPath, int homeworkId) {
try {
List<Person> personList = ExcelUtil.readExcel(excelPath);

@ -1,23 +1,12 @@
package net.educoder.ecsonar.task;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ZipUtil;
import com.github.junrar.Junrar;
import net.educoder.ecsonar.model.api.Person;
import net.educoder.ecsonar.model.api.SonarRequest;
import net.educoder.ecsonar.services.SonarService;
import net.educoder.ecsonar.utils.RarUtil;
import net.educoder.ecsonar.utils.UrlUtil;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* @Author: youys
@ -39,7 +28,6 @@ public class ReadExcelRunnable implements Runnable{
@Override
public void run() {
String uid = sonarRequest.getPersonList().get(0).getUid();
int homeworkId = sonarRequest.getHomeworkId();
String path = String.format("/tmp/%d/%s/", homeworkId, uid);
@ -47,69 +35,22 @@ public class ReadExcelRunnable implements Runnable{
if(!file.exists()){
file.mkdirs();
}
// for (Person person : sonarRequest.getPersonList()) {
// String[] split = StringUtils.split(person.getDownloadUrl(), "/");
//// String zipFilename = UrlUtil.getURLDecoderString(split[split.length - 1]);
// try {
// download(person.getDownloadUrl(),path);
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// log.info("开始调用");
String key = String.format("%d-%s", homeworkId, uid);
File scannerReport = new File(path + "/.scannerwork/report-task.txt");
if(scannerReport.exists()){
log.info("{} 已存在成功扫描标记,跳过: {}", key, scannerReport.getAbsolutePath());
return;
}
File scannerWork = new File(path + "/.scannerwork/");
if (scannerWork.exists()) {
log.info("{} 存在未完成的扫描工作目录,删除后重试: {}", key, scannerWork.getAbsolutePath());
FileUtil.del(scannerWork);
}
log.info("开始调用sonar:{}",key);
boolean success = sonarService.sonar(path,key);
if (!success) {
log.error("sonar扫描失败, uid:{}, key:{}, path:{}", uid, key, path);
}
}
private void download(String zipUrl, String zipPath) throws IOException {
log.info("下载文件: {}-{}", zipUrl, zipPath);
String cookie = "_educoder_session=0bea28f7a6db39cefb2fc3cbe715d840;";
URL url = new URL(zipUrl);
URLConnection conn = url.openConnection();
conn.setRequestProperty("Cookie", cookie);
conn.setDoInput(true);
String fileName = conn.getHeaderField("Content-Disposition");
if(StringUtils.isNotEmpty(fileName)){
fileName = new String(fileName.getBytes("ISO-8859-1"), "UTF-8");
fileName = UrlUtil.getURLDecoderString(fileName);
fileName = UrlUtil.getURLDecoderString(fileName);
fileName = fileName.substring(fileName.indexOf("filename=")+10);
// 如果存在分号 ;,截取前面部分
int semicolonIndex = fileName.indexOf(';');
if (semicolonIndex != -1) {
fileName = fileName.substring(0, semicolonIndex);
}
// 去掉可能的引号
fileName = fileName.replaceAll("\"", "");
zipPath = zipPath.substring(0, zipPath.lastIndexOf("/")+1) + fileName;
log.info("zipPath:{}",zipPath);
}
try (BufferedInputStream in = new BufferedInputStream(conn.getInputStream());
FileOutputStream fileOutputStream = new FileOutputStream(zipPath)) {
byte dataBuffer[] = new byte[1024];
int bytesRead;
while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
fileOutputStream.write(dataBuffer, 0, bytesRead);
}
} catch (IOException e) {
log.error("文件下载失败: {}-{}", zipUrl, zipPath);
throw e;
}
}
}

@ -2,9 +2,11 @@ package net.educoder.ecsonar.utils;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.RuntimeUtil;
import cn.hutool.core.util.ZipUtil;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.*;
@ -14,6 +16,8 @@ import java.util.HashSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* @Author: youys
@ -25,11 +29,21 @@ public class RarUtil {
private static final Set<String> SOURCE_EXTENSIONS = new HashSet<>(Arrays.asList(
".java", ".py", ".c", ".cc", ".cpp", ".cxx", ".h", ".hpp",
".js", ".jsx", ".ts", ".tsx", ".vue", ".go", ".php", ".rb",
".cs", ".m", ".mm", ".swift", ".kt", ".kts", ".scala"
".cs", ".m", ".mm", ".swift", ".kt", ".kts", ".scala", ".ets"
));
private static final Set<String> TARGET_EXTENSIONS = new HashSet<>(Arrays.asList(".cs", ".m", ".ets"));
private static final double TARGET_EXTENSION_RATIO = 0.8;
private static final Set<String> BUILD_AND_DEPENDENCY_DIRS = new HashSet<>(Arrays.asList(
"node_modules", "target", "dist", "build", "out", "bin", "obj",
".gradle", ".mvn", ".idea", ".vscode", "__pycache__",
".venv", "venv", "env", "site-packages", ".pytest_cache", ".mypy_cache",
".next", ".nuxt", ".output", "coverage", ".cache", ".parcel-cache", ".turbo",
".pnpm-store", "vendor", "Pods", "DerivedData", ".cxx", ".externalNativeBuild"
));
public static void unrar(String sourceFilePath, String outputDirectory) {
String s = RuntimeUtil.execForStr("unar", sourceFilePath,"-o", outputDirectory);
System.out.println("result====" + s);
@ -59,7 +73,10 @@ public class RarUtil {
public static void unzip(){
String sourceDir = "/tmp/2026051111";
String targetDir = "/tmp/202605112";
unzip(sourceDir, targetDir);
}
public static void unzip(String sourceDir, String targetDir){
File directory = new File(sourceDir);
File[] studentDirs = directory.listFiles();
if (studentDirs == null) {
@ -80,24 +97,29 @@ public class RarUtil {
String studentTargetDir = targetDir + "/" + studentId;
File[] zipFiles = studentDir.listFiles();
// File[] zipFiles = studentDir.listFiles((dir, name) -> name.endsWith(".zip"));
// if (zipFiles == null) {
// System.out.println("No zip files found for student: " + studentId);
// continue;
// }
if (zipFiles == null || zipFiles.length == 0) {
System.out.println("No archive files found for student: " + studentId);
continue;
}
for (File zipFile : zipFiles) {
if (!zipFile.isFile()) {
continue;
}
try {
if(zipFile.getName().endsWith(".zip")){
unzip(zipFile.getAbsolutePath(), studentTargetDir);
}else if(zipFile.getName().endsWith(".rar")){
String fileName = zipFile.getName().toLowerCase();
if(fileName.endsWith(".zip")){
unzipArchive(zipFile.getAbsolutePath(), studentTargetDir);
}else if(fileName.endsWith(".rar")){
unRar(zipFile.getAbsolutePath(), studentTargetDir);
}else{
RarUtil.un7Z(zipFile.getAbsolutePath(), studentTargetDir);
}
} catch (Exception e) {
System.out.println("Error occurred while unzipping file: " + zipFile.getName());
System.out.println("Error occurred while unzipping file: "
+ zipFile.getName()
+ ", path:" + zipFile.getAbsolutePath()
+ ", reason:" + e.getMessage());
}
}
@ -105,9 +127,91 @@ public class RarUtil {
}
}
private static void unzip(String zipFilePath, String targetDir) throws IOException {
public static void prepareForScan(String path) throws IOException {
normalizeWindowsPathFiles(Paths.get(path));
valid(path);
printSingleExtensionCodeDirs(path);
removeFullCode(path);
removeDirectory(Paths.get(path));
}
public static void normalizeWindowsPathFiles(Path rootPath) throws IOException {
Path normalizedRootPath = rootPath.toAbsolutePath().normalize();
Files.walkFileTree(normalizedRootPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String fileName = file.getFileName().toString();
if (!fileName.contains("\\")) {
return FileVisitResult.CONTINUE;
}
Path parent = file.getParent();
if (parent == null) {
return FileVisitResult.CONTINUE;
}
Path targetPath = parent.resolve(fileName.replace("\\", "/")).normalize();
if (!targetPath.startsWith(normalizedRootPath)) {
System.out.println("Skip unsafe normalized path: " + file.toAbsolutePath());
return FileVisitResult.CONTINUE;
}
if (fileName.endsWith("\\")) {
Files.createDirectories(targetPath);
Files.deleteIfExists(file);
return FileVisitResult.CONTINUE;
}
Path targetParent = targetPath.getParent();
if (targetParent != null) {
Files.createDirectories(targetParent);
}
Files.move(file, targetPath, StandardCopyOption.REPLACE_EXISTING);
System.out.println("Normalize windows path file: " + file.toAbsolutePath() + " -> " + targetPath);
return FileVisitResult.CONTINUE;
}
});
}
private static void unzipArchive(String zipFilePath, String targetDir) throws IOException {
System.out.println(zipFilePath + " to " + targetDir);
ZipUtil.unzip(zipFilePath,targetDir, Charset.forName("GBK"));
unzipArchiveWithGbkAndWindowsPath(zipFilePath, targetDir);
}
private static void unzipArchiveWithGbkAndWindowsPath(String zipFilePath, String targetDir) throws IOException {
Path targetPath = Paths.get(targetDir).toAbsolutePath().normalize();
Files.createDirectories(targetPath);
try (ZipInputStream zipInputStream = new ZipInputStream(
new BufferedInputStream(new FileInputStream(zipFilePath)), Charset.forName("GBK"))) {
ZipEntry entry;
byte[] buffer = new byte[8192];
while ((entry = zipInputStream.getNextEntry()) != null) {
String entryName = entry.getName().replace("\\", "/");
Path outputPath = targetPath.resolve(entryName).normalize();
if (!outputPath.startsWith(targetPath)) {
System.out.println("Skip unsafe zip entry: " + entry.getName());
zipInputStream.closeEntry();
continue;
}
if (entry.isDirectory() || entryName.endsWith("/")) {
Files.createDirectories(outputPath);
} else {
Path parent = outputPath.getParent();
if (parent != null) {
Files.createDirectories(parent);
}
try (FileOutputStream outputStream = new FileOutputStream(outputPath.toFile())) {
int len;
while ((len = zipInputStream.read(buffer)) > 0) {
outputStream.write(buffer, 0, len);
}
}
}
zipInputStream.closeEntry();
}
}
}
private static void unRar(String zipFilePath, String targetDir) throws Exception {
@ -254,12 +358,12 @@ public class RarUtil {
private static void removeDirectory(Path rootPath) throws IOException {
public static void removeDirectory(Path rootPath) throws IOException {
Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
String folderName = dir.getFileName().toString();
if (folderName.equals("node_modules") || folderName.equals("target") || folderName.equals("dist")) {
if (BUILD_AND_DEPENDENCY_DIRS.contains(folderName)) {
System.out.println("Deleting directory: " + dir.toAbsolutePath());
deleteDirectoryRecursively(dir);
@ -301,7 +405,7 @@ public class RarUtil {
}
public void finish(double targetExtensionRatio) {
for (String targetExtension : Arrays.asList(".cs", ".m", ".ets")) {
for (String targetExtension : TARGET_EXTENSIONS) {
int count = extensionCountMap.getOrDefault(targetExtension, 0);
double ratio = sourceFileCount == 0 ? 0 : count * 1.0 / sourceFileCount;
if (count > targetFileCount) {

@ -6,6 +6,8 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
/**
* Created by guange on 26/02/2017.
@ -41,6 +43,13 @@ public class SystemUtil {
* 退
*/
public static ExecuteResp executeAndGetExitStatus(String command) {
return executeAndGetExitStatus(command, 0, null, null);
}
public static ExecuteResp executeAndGetExitStatus(String command,
long timeout,
TimeUnit timeUnit,
Consumer<String> lineConsumer) {
ExecuteResp resp = new ExecuteResp();
logger.debug("execute: {}", command);
@ -52,13 +61,43 @@ public class SystemUtil {
pb.redirectErrorStream(true);
try {
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
out.append(line);
out.append(System.getProperty("line.separator"));
Thread outputThread = new Thread(() -> {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) {
out.append(line);
out.append(System.getProperty("line.separator"));
if (lineConsumer != null) {
lineConsumer.accept(line);
}
}
} catch (Exception e) {
logger.error("read process output failed: {}", command, e);
}
}, "command-output-reader");
outputThread.setDaemon(true);
outputThread.start();
boolean finished;
if (timeout > 0 && timeUnit != null) {
finished = process.waitFor(timeout, timeUnit);
} else {
exitStatus = process.waitFor();
finished = true;
}
if (!finished) {
process.destroyForcibly();
out.append("Command timeout after ")
.append(timeout)
.append(" ")
.append(timeUnit)
.append(System.getProperty("line.separator"));
logger.error("command timeout after {} {}: {}", timeout, timeUnit, command);
} else if (exitStatus == -1) {
exitStatus = process.exitValue();
}
exitStatus = process.waitFor();
outputThread.join(1000);
} catch (Exception e) {
e.printStackTrace();
@ -66,7 +105,7 @@ public class SystemUtil {
logger.error("executeAndGetExitStatus: ",e);
}
logger.debug("out: {}", out.toString());
logger.debug("out: {}", out);
resp.setOutput(out.toString().trim());
resp.setStatus(exitStatus);

@ -45,6 +45,9 @@ excel.template.path=template.xlsx
#sonar.url=http://127.0.0.1:9000
sonar.url=http://106.75.25.158:9000
sonar.scan.max-concurrent=3
sonar.scan.retry-times=3
sonar.scan.timeout-minutes=10
# token令牌
sonar.token=0253a518e824a976ea2f11aec17938cb0f8c0495

@ -7,6 +7,7 @@ import net.educoder.ecsonar.model.api.Person;
import net.educoder.ecsonar.services.ReportService;
import net.educoder.ecsonar.services.SonarService;
import net.educoder.ecsonar.utils.ExcelUtil;
import net.educoder.ecsonar.utils.RarUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -64,12 +65,19 @@ public class EcsonarApplicationTests {
}
@Test
public void scanExcel(){
public void scanExcel() throws Exception {
CountDownLatch countDownLatch = new CountDownLatch(1);
String excelPath = "/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/毕业设计-湘潭大学网络空间安全专业2026届.xlsx";
// String excelPath = "/Users/youyongsheng/Desktop/aaaa.xlsx";
sonarService.scanExcel(excelPath);
int downloadHomeworkId = 202605131;
int scanHomeworkId = 202605132;
String downloadDirectory = String.format("/tmp/%d", downloadHomeworkId);
String scanDirectory = String.format("/tmp/%d", scanHomeworkId);
String excelPath = "/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/2026届湘潭大学计算机科学与技术系.xlsx";
sonarService.downloadExcelArchives(excelPath, downloadHomeworkId);
RarUtil.unzip(downloadDirectory, scanDirectory);
RarUtil.prepareForScan(scanDirectory);
sonarService.scanExcel(excelPath, scanHomeworkId);
try {
countDownLatch.await();
@ -81,13 +89,13 @@ public class EcsonarApplicationTests {
@Test
public void writeExcel() throws Exception {
int homeworkId = 202605112;
String directory = "/tmp/202605112/";
int homeworkId = 202605132;
String directory = "/tmp/202605132/";
File file = new File(directory);
File[] files = file.listFiles();
System.out.println("学生数:" + files.length);
List<Person> personList = ExcelUtil.readExcel("/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/毕业设计-湘潭大学网络空间安全专业2026届.xlsx");
List<Person> personList = ExcelUtil.readExcel("/Users/youyongsheng/Desktop/ZQ/质量分析/2026-05-11/2026届湘潭大学计算机科学与技术系.xlsx");
Map<String, List<Person>> collect = personList.stream().collect(Collectors.groupingBy(Person::getUid));
@ -100,7 +108,7 @@ public class EcsonarApplicationTests {
Metrics metrics = reportService.getMetrics(projectName);
String templatePath = this.getClass().getClassLoader().getResource("template1.xlsx").getPath();
String outPath = "/Users/youyongsheng/Desktop/cccc.xlsx";
String outPath = "/Users/youyongsheng/Desktop/dddd3.xlsx";
System.out.println("第"+(i++)+"个学生," + uid);
if (CollectionUtils.isEmpty(collect.get(uid))) {

Loading…
Cancel
Save