|
|
|
|
@ -2,9 +2,11 @@ package net.educoder.ecsonar.utils;
|
|
|
|
|
|
|
|
|
|
import cn.hutool.core.io.FileUtil;
|
|
|
|
|
import cn.hutool.core.util.RuntimeUtil;
|
|
|
|
|
import cn.hutool.core.util.ZipUtil;
|
|
|
|
|
|
|
|
|
|
import java.io.BufferedInputStream;
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.FileInputStream;
|
|
|
|
|
import java.io.FileOutputStream;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.nio.charset.Charset;
|
|
|
|
|
import java.nio.file.*;
|
|
|
|
|
@ -14,6 +16,8 @@ import java.util.HashSet;
|
|
|
|
|
import java.util.HashMap;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.Set;
|
|
|
|
|
import java.util.zip.ZipEntry;
|
|
|
|
|
import java.util.zip.ZipInputStream;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @Author: youys
|
|
|
|
|
@ -25,11 +29,21 @@ public class RarUtil {
|
|
|
|
|
private static final Set<String> SOURCE_EXTENSIONS = new HashSet<>(Arrays.asList(
|
|
|
|
|
".java", ".py", ".c", ".cc", ".cpp", ".cxx", ".h", ".hpp",
|
|
|
|
|
".js", ".jsx", ".ts", ".tsx", ".vue", ".go", ".php", ".rb",
|
|
|
|
|
".cs", ".m", ".mm", ".swift", ".kt", ".kts", ".scala"
|
|
|
|
|
".cs", ".m", ".mm", ".swift", ".kt", ".kts", ".scala", ".ets"
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
private static final Set<String> TARGET_EXTENSIONS = new HashSet<>(Arrays.asList(".cs", ".m", ".ets"));
|
|
|
|
|
|
|
|
|
|
private static final double TARGET_EXTENSION_RATIO = 0.8;
|
|
|
|
|
|
|
|
|
|
private static final Set<String> BUILD_AND_DEPENDENCY_DIRS = new HashSet<>(Arrays.asList(
|
|
|
|
|
"node_modules", "target", "dist", "build", "out", "bin", "obj",
|
|
|
|
|
".gradle", ".mvn", ".idea", ".vscode", "__pycache__",
|
|
|
|
|
".venv", "venv", "env", "site-packages", ".pytest_cache", ".mypy_cache",
|
|
|
|
|
".next", ".nuxt", ".output", "coverage", ".cache", ".parcel-cache", ".turbo",
|
|
|
|
|
".pnpm-store", "vendor", "Pods", "DerivedData", ".cxx", ".externalNativeBuild"
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
public static void unrar(String sourceFilePath, String outputDirectory) {
|
|
|
|
|
String s = RuntimeUtil.execForStr("unar", sourceFilePath,"-o", outputDirectory);
|
|
|
|
|
System.out.println("result====" + s);
|
|
|
|
|
@ -59,7 +73,10 @@ public class RarUtil {
|
|
|
|
|
public static void unzip(){
|
|
|
|
|
String sourceDir = "/tmp/2026051111";
|
|
|
|
|
String targetDir = "/tmp/202605112";
|
|
|
|
|
unzip(sourceDir, targetDir);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void unzip(String sourceDir, String targetDir){
|
|
|
|
|
File directory = new File(sourceDir);
|
|
|
|
|
File[] studentDirs = directory.listFiles();
|
|
|
|
|
if (studentDirs == null) {
|
|
|
|
|
@ -80,24 +97,29 @@ public class RarUtil {
|
|
|
|
|
String studentTargetDir = targetDir + "/" + studentId;
|
|
|
|
|
|
|
|
|
|
File[] zipFiles = studentDir.listFiles();
|
|
|
|
|
// File[] zipFiles = studentDir.listFiles((dir, name) -> name.endsWith(".zip"));
|
|
|
|
|
// if (zipFiles == null) {
|
|
|
|
|
// System.out.println("No zip files found for student: " + studentId);
|
|
|
|
|
// continue;
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
if (zipFiles == null || zipFiles.length == 0) {
|
|
|
|
|
System.out.println("No archive files found for student: " + studentId);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (File zipFile : zipFiles) {
|
|
|
|
|
if (!zipFile.isFile()) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if(zipFile.getName().endsWith(".zip")){
|
|
|
|
|
unzip(zipFile.getAbsolutePath(), studentTargetDir);
|
|
|
|
|
}else if(zipFile.getName().endsWith(".rar")){
|
|
|
|
|
String fileName = zipFile.getName().toLowerCase();
|
|
|
|
|
if(fileName.endsWith(".zip")){
|
|
|
|
|
unzipArchive(zipFile.getAbsolutePath(), studentTargetDir);
|
|
|
|
|
}else if(fileName.endsWith(".rar")){
|
|
|
|
|
unRar(zipFile.getAbsolutePath(), studentTargetDir);
|
|
|
|
|
}else{
|
|
|
|
|
RarUtil.un7Z(zipFile.getAbsolutePath(), studentTargetDir);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
System.out.println("Error occurred while unzipping file: " + zipFile.getName());
|
|
|
|
|
System.out.println("Error occurred while unzipping file: "
|
|
|
|
|
+ zipFile.getName()
|
|
|
|
|
+ ", path:" + zipFile.getAbsolutePath()
|
|
|
|
|
+ ", reason:" + e.getMessage());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -105,9 +127,91 @@ public class RarUtil {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void unzip(String zipFilePath, String targetDir) throws IOException {
|
|
|
|
|
public static void prepareForScan(String path) throws IOException {
|
|
|
|
|
normalizeWindowsPathFiles(Paths.get(path));
|
|
|
|
|
valid(path);
|
|
|
|
|
printSingleExtensionCodeDirs(path);
|
|
|
|
|
removeFullCode(path);
|
|
|
|
|
removeDirectory(Paths.get(path));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void normalizeWindowsPathFiles(Path rootPath) throws IOException {
|
|
|
|
|
Path normalizedRootPath = rootPath.toAbsolutePath().normalize();
|
|
|
|
|
Files.walkFileTree(normalizedRootPath, new SimpleFileVisitor<Path>() {
|
|
|
|
|
@Override
|
|
|
|
|
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
|
|
|
|
String fileName = file.getFileName().toString();
|
|
|
|
|
if (!fileName.contains("\\")) {
|
|
|
|
|
return FileVisitResult.CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Path parent = file.getParent();
|
|
|
|
|
if (parent == null) {
|
|
|
|
|
return FileVisitResult.CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Path targetPath = parent.resolve(fileName.replace("\\", "/")).normalize();
|
|
|
|
|
if (!targetPath.startsWith(normalizedRootPath)) {
|
|
|
|
|
System.out.println("Skip unsafe normalized path: " + file.toAbsolutePath());
|
|
|
|
|
return FileVisitResult.CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (fileName.endsWith("\\")) {
|
|
|
|
|
Files.createDirectories(targetPath);
|
|
|
|
|
Files.deleteIfExists(file);
|
|
|
|
|
return FileVisitResult.CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Path targetParent = targetPath.getParent();
|
|
|
|
|
if (targetParent != null) {
|
|
|
|
|
Files.createDirectories(targetParent);
|
|
|
|
|
}
|
|
|
|
|
Files.move(file, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
|
|
|
|
System.out.println("Normalize windows path file: " + file.toAbsolutePath() + " -> " + targetPath);
|
|
|
|
|
return FileVisitResult.CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void unzipArchive(String zipFilePath, String targetDir) throws IOException {
|
|
|
|
|
System.out.println(zipFilePath + " to " + targetDir);
|
|
|
|
|
ZipUtil.unzip(zipFilePath,targetDir, Charset.forName("GBK"));
|
|
|
|
|
unzipArchiveWithGbkAndWindowsPath(zipFilePath, targetDir);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void unzipArchiveWithGbkAndWindowsPath(String zipFilePath, String targetDir) throws IOException {
|
|
|
|
|
Path targetPath = Paths.get(targetDir).toAbsolutePath().normalize();
|
|
|
|
|
Files.createDirectories(targetPath);
|
|
|
|
|
|
|
|
|
|
try (ZipInputStream zipInputStream = new ZipInputStream(
|
|
|
|
|
new BufferedInputStream(new FileInputStream(zipFilePath)), Charset.forName("GBK"))) {
|
|
|
|
|
ZipEntry entry;
|
|
|
|
|
byte[] buffer = new byte[8192];
|
|
|
|
|
while ((entry = zipInputStream.getNextEntry()) != null) {
|
|
|
|
|
String entryName = entry.getName().replace("\\", "/");
|
|
|
|
|
Path outputPath = targetPath.resolve(entryName).normalize();
|
|
|
|
|
if (!outputPath.startsWith(targetPath)) {
|
|
|
|
|
System.out.println("Skip unsafe zip entry: " + entry.getName());
|
|
|
|
|
zipInputStream.closeEntry();
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (entry.isDirectory() || entryName.endsWith("/")) {
|
|
|
|
|
Files.createDirectories(outputPath);
|
|
|
|
|
} else {
|
|
|
|
|
Path parent = outputPath.getParent();
|
|
|
|
|
if (parent != null) {
|
|
|
|
|
Files.createDirectories(parent);
|
|
|
|
|
}
|
|
|
|
|
try (FileOutputStream outputStream = new FileOutputStream(outputPath.toFile())) {
|
|
|
|
|
int len;
|
|
|
|
|
while ((len = zipInputStream.read(buffer)) > 0) {
|
|
|
|
|
outputStream.write(buffer, 0, len);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
zipInputStream.closeEntry();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void unRar(String zipFilePath, String targetDir) throws Exception {
|
|
|
|
|
@ -254,12 +358,12 @@ public class RarUtil {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static void removeDirectory(Path rootPath) throws IOException {
|
|
|
|
|
public static void removeDirectory(Path rootPath) throws IOException {
|
|
|
|
|
Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() {
|
|
|
|
|
@Override
|
|
|
|
|
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
|
|
|
|
|
String folderName = dir.getFileName().toString();
|
|
|
|
|
if (folderName.equals("node_modules") || folderName.equals("target") || folderName.equals("dist")) {
|
|
|
|
|
if (BUILD_AND_DEPENDENCY_DIRS.contains(folderName)) {
|
|
|
|
|
System.out.println("Deleting directory: " + dir.toAbsolutePath());
|
|
|
|
|
deleteDirectoryRecursively(dir);
|
|
|
|
|
|
|
|
|
|
@ -301,7 +405,7 @@ public class RarUtil {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void finish(double targetExtensionRatio) {
|
|
|
|
|
for (String targetExtension : Arrays.asList(".cs", ".m", ".ets")) {
|
|
|
|
|
for (String targetExtension : TARGET_EXTENSIONS) {
|
|
|
|
|
int count = extensionCountMap.getOrDefault(targetExtension, 0);
|
|
|
|
|
double ratio = sourceFileCount == 0 ? 0 : count * 1.0 / sourceFileCount;
|
|
|
|
|
if (count > targetFileCount) {
|
|
|
|
|
|