diff --git a/app/controllers/myshixuns_controller.rb b/app/controllers/myshixuns_controller.rb index 350ee581f..7fd3547b9 100644 --- a/app/controllers/myshixuns_controller.rb +++ b/app/controllers/myshixuns_controller.rb @@ -265,6 +265,7 @@ class MyshixunsController < ApplicationController uid_logger_dubug("-- game build: file update #{@sec_key}, record id is #{record.id}, time is **** #{Time.now.strftime("%Y-%m-%d %H:%M:%S.%L")}") end # 隐藏代码文件 和 VNC的都不需要走版本库 + uid_logger_dubug("#######enter_#{@myshixun.identifier}: #{@hide_code}") unless @hide_code || (@myshixun.shixun&.vnc_evaluate && params[:evaluate].present?) # 远程版本库文件内容 last_content = GitService.file_content(repo_path: @repo_path, path: path)["content"] @@ -275,8 +276,8 @@ class MyshixunsController < ApplicationController else params[:content] end - uid_logger_dubug("###11222333####{content}") - uid_logger_dubug("###222333####{last_content}") + uid_logger_dubug("content_#{@myshixun.identifier}: #{content}") + uid_logger_dubug("###last_content_#{@myshixun.identifier}####{last_content}") if content != last_content @content_modified = 1 @@ -286,6 +287,7 @@ class MyshixunsController < ApplicationController message = params[:evaluate] == 0 ? "System automatically submitted" : "User submitted" uid_logger_dubug("112233#{author_name}") uid_logger_dubug("112233#{author_email}") + uid_logger_dubug("daiao_debug_#{@myshixun.identifier}: #{@repo_path}: #{path}; #{message}; #{content}; ") @content = GitService.update_file(repo_path: @repo_path, file_path: path, message: message, diff --git a/app/services/subjects/course_used_info_service.rb b/app/services/subjects/course_used_info_service.rb index 5ae5abe1c..296e1ed94 100644 --- a/app/services/subjects/course_used_info_service.rb +++ b/app/services/subjects/course_used_info_service.rb @@ -22,7 +22,7 @@ class Subjects::CourseUsedInfoService < ApplicationService # choice_shixun_frequency: 选用该课程实训的次数 course_info = [] schools.find_in_batches do |s| - Parallel.each(s) do |school| + s.each do |school| name = school.name course_count = school.course_count student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size diff --git a/app/services/subjects/shixun_used_info_service.rb b/app/services/subjects/shixun_used_info_service.rb index 5fa99a120..bd145e622 100644 --- a/app/services/subjects/shixun_used_info_service.rb +++ b/app/services/subjects/shixun_used_info_service.rb @@ -11,7 +11,7 @@ class Subjects::ShixunUsedInfoService < ApplicationService position = stage.position shixuns = stage.shixuns.includes(myshixuns: :games, homework_commons: :course) shixuns.find_in_batches(batch_size: 1000) do |s| - Parallel.each_with_index(s, in_processes: 2) do |shixun, index| + s.each_with_index do |shixun, index| stage = "#{position}-#{index+1}" name = shixun.name myshixuns = shixun.myshixuns diff --git a/app/services/subjects/user_used_info_service.rb b/app/services/subjects/user_used_info_service.rb index 78dbfe8e9..2e6594575 100644 --- a/app/services/subjects/user_used_info_service.rb +++ b/app/services/subjects/user_used_info_service.rb @@ -13,7 +13,7 @@ class Subjects::UserUsedInfoService < ApplicationService users_info = [] users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2}, users: {is_test: false}) users.find_in_batches(batch_size: 500) do |u| - Parallel.each(u, in_processes: 2) do |user| + u.each do |user| myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)} name = "#{user.lastname}#{user.firstname}" passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size diff --git a/lib/tasks/statistic_subject_info.rake b/lib/tasks/statistic_subject_info.rake index d188011a7..71fb8ed36 100644 --- a/lib/tasks/statistic_subject_info.rake +++ b/lib/tasks/statistic_subject_info.rake @@ -5,14 +5,14 @@ namespace :subjects do puts("---------------------data_statistic_begin") Rails.logger.info("---------------------data_statistic_begin") subjects = Subject.where(status: 2, hidden: 0) + if ENV['subject_id'].present? + subjects = subjects.where(id:ENV['subject_id']) + end column_value = "subject_id, study_count, course_study_count, initiative_study, passed_count, course_used_count, " + "school_used_count, created_at, updated_at" subjects.find_in_batches(batch_size: 50) do |s| - str = "" - buffer_size = 0 - Parallel.each(s, in_processes: 4) do |subject| - buffer_size += 1 - str += ", " unless str.empty? + str = [] + Parallel.each_with_index(s, in_threads: 4) do |subject, index| puts("---------------------data_statistic: #{subject.id}") Rails.logger.info("---------------------data_statistic: #{subject.id}") data = Subjects::DataStatisticService.new(subject) @@ -20,25 +20,19 @@ namespace :subjects do next if study_count == 0 course_study_count = data.course_study_count initiative_study = study_count - course_study_count - str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " + + str << ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " + "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - puts "buffer_size: #{buffer_size}; subject_id: #{subject.id}; s: #{s.last}; subject: #{subject == s.last};" + puts "index: #{index}; worker_number: #{Parallel.worker_number}" puts "####str: #{str}" - if buffer_size == 1000 || subject == s.last - sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 - end + end + if str.size > 0 + sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str.uniq.join(",")}" + puts sql + ActiveRecord::Base.connection.execute sql end end - if buffer_size > 0 - sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - end + puts("---------------------data_statistic_end") Rails.logger.info("---------------------data_statistic_end") end @@ -47,38 +41,37 @@ namespace :subjects do puts("---------------------course_info_statistic_begin") Rails.logger.info("---------------------course_info_statistic_begin") subjects = Subject.where(status: 2, hidden: 0) - str = "" - buffer_size = 0 column_value = "subject_id, school_id, school_name, course_count, student_count, choice_shixun_num, " + "choice_shixun_frequency, created_at, updated_at" - - subjects.find_in_batches(batch_size: 50) do |s| - Parallel.each(s, in_processes: 4) do |subject| + if ENV['subject_id'].present? + subjects = subjects.where(id:ENV['subject_id']) + end + subjects.find_in_batches(batch_size: 20) do |s| + str = [] + Parallel.each(s, in_processes: 6) do |subject| puts("---------------------course_info_statistic: #{subject.id}") Rails.logger.info("---------------------course_info_statistic: #{subject.id}") data = Subjects::CourseUsedInfoService.call(subject) data.each do |key| next if key[:school_id].nil? - str += ", " unless str.empty? - str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " + + str << ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " + "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 || key == data.last - sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 - end + # if str.size == 1000 + # sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str.uniq.join(",")}" + # str_c = str + # puts sql + # ActiveRecord::Base.connection.execute sql + # str -= str_c + # end + end + if str.size > 0 + sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str.uniq.join(",")}" + puts sql + ActiveRecord::Base.connection.execute sql end end end - if buffer_size > 0 - sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - end puts("---------------------course_info_statistic_end") Rails.logger.info("---------------------course_info_statistic_end") end @@ -87,38 +80,35 @@ namespace :subjects do puts("---------------------shixun_info_statistic_begin") Rails.logger.info("---------------------shixun_info_statistic_begin") subjects = Subject.where(status: 2, hidden: 0) - str = "" - buffer_size = 0 + if ENV['subject_id'].present? + subjects = subjects.where(id:ENV['subject_id']) + end column_value = "subject_id, shixun_id, stage, shixun_name, challenge_count, course_count, " + "school_count, used_count, passed_count, evaluate_count, passed_ave_time, created_at, updated_at" - subjects.find_in_batches(batch_size: 50) do |s| - Parallel.each_with_index(s, in_processes: 4) do |subject| + subjects.find_in_batches(batch_size: 20) do |s| + Parallel.each_with_index(s, in_processes: 4) do |subject, index, str = []| puts("---------------------shixun_info_statistic: #{subject.id}") Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}") data = Subjects::ShixunUsedInfoService.call(subject) data.each do |key| next if key[:shixun_id].nil? - str += ", " unless str.empty? - str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " + + str << ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " + "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " + "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 || key == data.last - sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 - end + # if str.size == 1000 + # sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str.join(",")}" + # puts sql + # ActiveRecord::Base.connection.execute sql + # end + end + if str.size > 0 + sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str.join(",")}" + puts sql + ActiveRecord::Base.connection.execute sql end end end - if buffer_size > 0 - sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - end puts("---------------------shixun_info_statistic_end") Rails.logger.info("---------------------shixun_info_statistic_end") end @@ -127,36 +117,29 @@ namespace :subjects do puts("---------------------user_info_statistic_begin") Rails.logger.info("---------------------user_info_statistic_begin") subjects = Subject.where(status: 2, hidden: 0) - str = "" - buffer_size = 0 column_value = "user_id, subject_id, username, passed_myshixun_count, passed_games_count, " + "code_line_count, evaluate_count, cost_time, created_at, updated_at" - - subjects.find_in_batches(batch_size: 50) do |s| - Parallel.each_with_index(s, in_processes: 4) do |subject, index| + subjects.find_in_batches(batch_size: 20) do |s| + Parallel.each_with_index(s, in_processes: 4) do |subject, index, str = []| puts("---------------------user_info_statistic: #{subject.id}") data = Subjects::UserUsedInfoService.call(subject) data.each do |key| - next if key[:user_id].nil? - str += ", " unless str.empty? - str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " + + str << ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " + "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 || (key == data.last) - sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}" - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 - end + # if str.size == 1000 + # sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str.join(",")}" + # ActiveRecord::Base.connection.execute sql + # str = [] + # end + end + if str.size > 0 + sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str.join(",")}" + puts sql + ActiveRecord::Base.connection.execute sql end end end - if buffer_size > 0 - sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - end puts("---------------------user_info_statistic_end") Rails.logger.info("---------------------user_info_statistic_end") end