diff --git a/Gemfile b/Gemfile index 3bfe246a4..859090feb 100644 --- a/Gemfile +++ b/Gemfile @@ -109,3 +109,5 @@ gem 'request_store' # 敏感词汇 gem 'harmonious_dictionary', '~> 0.0.1' + +gem 'parallel', '~> 1.19', '>= 1.19.1' diff --git a/app/controllers/item_banks_controller.rb b/app/controllers/item_banks_controller.rb index f108c1fad..86b1523c3 100644 --- a/app/controllers/item_banks_controller.rb +++ b/app/controllers/item_banks_controller.rb @@ -43,9 +43,8 @@ class ItemBanksController < ApplicationController ApplyAction.where(container_type: "ItemBank", container_id: @item.id).destroy_all if @item.item_type == "PROGRAM" @item.container&.destroy! - else - @item.destroy! end + @item.destroy! render_ok end end diff --git a/app/services/subjects/course_used_info_service.rb b/app/services/subjects/course_used_info_service.rb index 2561f3aa4..5ae5abe1c 100644 --- a/app/services/subjects/course_used_info_service.rb +++ b/app/services/subjects/course_used_info_service.rb @@ -21,17 +21,19 @@ class Subjects::CourseUsedInfoService < ApplicationService # choice_shixun_num: 选用该课程实训的个数(去重) # choice_shixun_frequency: 选用该课程实训的次数 course_info = [] - schools.map do |school| - name = school.name - course_count = school.course_count - student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size - shixun_ids = school.courses.joins(homework_commons: :homework_commons_shixun) - .where(homework_commons: {id: homework_common_ids}) - .pluck("homework_commons_shixuns.shixun_id") - choice_shixun_frequency = shixun_ids.size - choice_shixun_num = shixun_ids.uniq.size - course_info << {school_id: school.id, school_name: name, course_count: course_count, student_count: student_count, - choice_shixun_num: choice_shixun_num, choice_shixun_frequency: choice_shixun_frequency} + schools.find_in_batches do |s| + Parallel.each(s) do |school| + name = school.name + course_count = school.course_count + student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size + shixun_ids = school.courses.joins(homework_commons: :homework_commons_shixun) + .where(homework_commons: {id: homework_common_ids}) + .pluck("homework_commons_shixuns.shixun_id") + choice_shixun_frequency = shixun_ids.size + choice_shixun_num = shixun_ids.uniq.size + course_info << {school_id: school.id, school_name: name, course_count: course_count, student_count: student_count, + choice_shixun_num: choice_shixun_num, choice_shixun_frequency: choice_shixun_frequency} + end end course_info end diff --git a/app/services/subjects/shixun_used_info_service.rb b/app/services/subjects/shixun_used_info_service.rb index dfd8485e0..5fa99a120 100644 --- a/app/services/subjects/shixun_used_info_service.rb +++ b/app/services/subjects/shixun_used_info_service.rb @@ -10,20 +10,23 @@ class Subjects::ShixunUsedInfoService < ApplicationService stages.each do |stage| position = stage.position shixuns = stage.shixuns.includes(myshixuns: :games, homework_commons: :course) - shixuns.each_with_index do |shixun, index| - stage = "#{position}-#{index+1}" - name = shixun.name - myshixuns = shixun.myshixuns - challenge_count = shixun.challenges_count - course_count = shixun.homework_commons.map{|hc| hc.course_id}.uniq.size - school_count = shixun.homework_commons.map{|hc| hc.course&.school_id}.uniq.size - used_count = shixun.myshixuns_count - passed_count = myshixuns.select{|m| m.status == 1}.size - evaluate_count = myshixuns.map{|m| m.output_times }.sum - passed_ave_time = passed_count > 0 ? myshixuns.map{|m| m.total_cost_time}.sum : 0 - shixun_infos << {stage: stage, name: name, challenge_count: challenge_count, course_count: course_count, - school_count: school_count, used_count: used_count, passed_count: passed_count, - evaluate_count: evaluate_count, passed_ave_time: passed_ave_time, shixun_id: shixun.id} + shixuns.find_in_batches(batch_size: 1000) do |s| + Parallel.each_with_index(s, in_processes: 2) do |shixun, index| + stage = "#{position}-#{index+1}" + name = shixun.name + myshixuns = shixun.myshixuns + challenge_count = shixun.challenges_count + course_count = shixun.homework_commons.map{|hc| hc.course_id}.uniq.size + school_count = shixun.homework_commons.map{|hc| hc.course&.school_id}.uniq.size + used_count = shixun.myshixuns_count + passed_count = myshixuns.select{|m| m.status == 1}.size + evaluate_count = myshixuns.map{|m| m.output_times }.sum + passed_ave_time = passed_count > 0 ? myshixuns.map{|m| m.total_cost_time}.sum : 0 + shixun_infos << {stage: stage, name: name, challenge_count: challenge_count, course_count: course_count, + school_count: school_count, used_count: used_count, passed_count: passed_count, + evaluate_count: evaluate_count, passed_ave_time: passed_ave_time, shixun_id: shixun.id} + + end end end shixun_infos diff --git a/app/services/subjects/user_used_info_service.rb b/app/services/subjects/user_used_info_service.rb index 52d6be005..78dbfe8e9 100644 --- a/app/services/subjects/user_used_info_service.rb +++ b/app/services/subjects/user_used_info_service.rb @@ -1,6 +1,8 @@ class Subjects::UserUsedInfoService < ApplicationService + require 'parallel' attr_reader :subject, :shixun_ids + def initialize(subject) @subject = subject @shixun_ids = subject.shixuns.pluck(:id) @@ -9,19 +11,22 @@ class Subjects::UserUsedInfoService < ApplicationService def call users_info = [] - users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2}) - users.find_each do |user| - myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)} - name = "#{user.lastname}#{user.firstname}" - passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size - passed_games_count = myshixuns.map{|m| m.games.select{|g| g.status == 2}.size }.size - code_line_count = 0 - evaluate_count = myshixuns.map{|m| m.output_times }.sum - cost_time = myshixuns.map{|m|m.total_cost_time }.sum - users_info << {user_id: user.id, name: name, passed_myshixun_count: passed_myshixun_count, - passed_games_count: passed_games_count, code_line_count: code_line_count, - evaluate_count: evaluate_count, cost_time: cost_time} + users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2}, users: {is_test: false}) + users.find_in_batches(batch_size: 500) do |u| + Parallel.each(u, in_processes: 2) do |user| + myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)} + name = "#{user.lastname}#{user.firstname}" + passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size + passed_games_count = myshixuns.map{|m| m.games.select{|g| g.status == 2}.size }.size + code_line_count = 0 + evaluate_count = myshixuns.map{|m| m.output_times }.sum + cost_time = myshixuns.map{|m|m.total_cost_time }.sum + users_info << {user_id: user.id, name: name, passed_myshixun_count: passed_myshixun_count, + passed_games_count: passed_games_count, code_line_count: code_line_count, + evaluate_count: evaluate_count, cost_time: cost_time} + end end + users_info end diff --git a/lib/tasks/statistic_subject_info.rake b/lib/tasks/statistic_subject_info.rake index 68d75293d..259824174 100644 --- a/lib/tasks/statistic_subject_info.rake +++ b/lib/tasks/statistic_subject_info.rake @@ -4,30 +4,32 @@ namespace :subjects do task data_statistic: :environment do puts("---------------------data_statistic_begin") Rails.logger.info("---------------------data_statistic_begin") - subjects = Subject.where(status: 2) + subjects = Subject.where(status: 2, hidden: 0) str = "" buffer_size = 0 column_value = "subject_id, study_count, course_study_count, initiative_study, passed_count, course_used_count, " + "school_used_count, created_at, updated_at" - subjects.find_each do |subject| - puts("---------------------data_statistic: #{subject.id}") - Rails.logger.info("---------------------data_statistic: #{subject.id}") - data = Subjects::DataStatisticService.new(subject) - study_count = data.study_count - next if study_count == 0 - course_study_count = data.course_study_count - initiative_study = study_count - course_study_count - str += ", " unless str.empty? - str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " + - "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " + - "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 - sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 + subjects.find_in_batches(batch_size: 50) do |s| + Parallel.each(s, in_processes: 4) do |subject| + puts("---------------------data_statistic: #{subject.id}") + Rails.logger.info("---------------------data_statistic: #{subject.id}") + data = Subjects::DataStatisticService.new(subject) + study_count = data.study_count + next if study_count == 0 + course_study_count = data.course_study_count + initiative_study = study_count - course_study_count + str += ", " unless str.empty? + str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " + + "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " + + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") + buffer_size += 1 + if buffer_size == 1000 + sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}" + puts sql + ActiveRecord::Base.connection.execute sql + str = "" + buffer_size = 0 + end end end if buffer_size > 0 @@ -42,28 +44,31 @@ namespace :subjects do task course_info_statistic: :environment do puts("---------------------course_info_statistic_begin") Rails.logger.info("---------------------course_info_statistic_begin") - subjects = Subject.where(status: 2) + subjects = Subject.where(status: 2, hidden: 0) str = "" buffer_size = 0 column_value = "subject_id, school_id, school_name, course_count, student_count, choice_shixun_num, " + "choice_shixun_frequency, created_at, updated_at" - subjects.find_each do |subject| - puts("---------------------course_info_statistic: #{subject.id}") - Rails.logger.info("---------------------course_info_statistic: #{subject.id}") - data = Subjects::CourseUsedInfoService.call(subject) - data.each do |key| - next if key[:school_id].nil? - str += ", " unless str.empty? - str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " + - "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " + - "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 - sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 + + subjects.find_in_batches(batch_size: 50) do |s| + Parallel.each(s, in_processes: 4) do |subject| + puts("---------------------course_info_statistic: #{subject.id}") + Rails.logger.info("---------------------course_info_statistic: #{subject.id}") + data = Subjects::CourseUsedInfoService.call(subject) + Parallel.map(data) do |key| + next if key[:school_id].nil? + str += ", " unless str.empty? + str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " + + "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " + + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") + buffer_size += 1 + if buffer_size == 1000 + sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}" + puts sql + ActiveRecord::Base.connection.execute sql + str = "" + buffer_size = 0 + end end end end @@ -79,29 +84,31 @@ namespace :subjects do task shixun_info_statistic: :environment do puts("---------------------shixun_info_statistic_begin") Rails.logger.info("---------------------shixun_info_statistic_begin") - subjects = Subject.where(status: 2) + subjects = Subject.where(status: 2, hidden: 0) str = "" buffer_size = 0 column_value = "subject_id, shixun_id, stage, shixun_name, challenge_count, course_count, " + "school_count, used_count, passed_count, evaluate_count, passed_ave_time, created_at, updated_at" - subjects.find_each(batch_size: 100) do |subject| - puts("---------------------shixun_info_statistic: #{subject.id}") - Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}") - data = Subjects::ShixunUsedInfoService.call(subject) - data.each do |key| - next if key[:shixun_id].nil? - str += ", " unless str.empty? - str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " + - "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " + - "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " + - "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 - sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}" - puts sql - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 + subjects.find_in_batches(batch_size: 50) do |s| + Parallel.each(s, in_processes: 4) do |subject| + puts("---------------------shixun_info_statistic: #{subject.id}") + Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}") + data = Subjects::ShixunUsedInfoService.call(subject) + data.each do |key| + next if key[:shixun_id].nil? + str += ", " unless str.empty? + str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " + + "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " + + "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " + + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") + buffer_size += 1 + if buffer_size == 1000 + sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}" + puts sql + ActiveRecord::Base.connection.execute sql + str = "" + buffer_size = 0 + end end end end @@ -117,26 +124,29 @@ namespace :subjects do task user_info_statistic: :environment do puts("---------------------user_info_statistic_begin") Rails.logger.info("---------------------user_info_statistic_begin") - subjects = Subject.where(status: 2) + subjects = Subject.where(status: 2, hidden: 0) str = "" buffer_size = 0 column_value = "user_id, subject_id, username, passed_myshixun_count, passed_games_count, " + "code_line_count, evaluate_count, cost_time, created_at, updated_at" - subjects.find_each(batch_size: 50) do |subject| - puts("---------------------user_info_statistic: #{subject.id}") - data = Subjects::UserUsedInfoService.call(subject) - data.each do |key| - next if key[:user_id].nil? - str += ", " unless str.empty? - str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name]}', '#{key[:passed_myshixun_count]}', " + - "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " + - "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") - buffer_size += 1 - if buffer_size == 1000 - sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}" - ActiveRecord::Base.connection.execute sql - str = "" - buffer_size = 0 + + subjects.find_in_batches(batch_size: 50) do |s| + Parallel.each(s, in_processes: 4) do |subject| + puts("---------------------user_info_statistic: #{subject.id}") + data = Subjects::UserUsedInfoService.call(subject) + data.each do |key| + next if key[:user_id].nil? + str += ", " unless str.empty? + str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " + + "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " + + "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')") + buffer_size += 1 + if buffer_size == 1000 + sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}" + ActiveRecord::Base.connection.execute sql + str = "" + buffer_size = 0 + end end end end