Merge branch 'dev_aliyun' of http://bdgit.educoder.net/Hjqreturn/educoder into dev_aliyun

Gemfile

@@ -109,3 +109,5 @@ gem 'request_store'
 
 # 敏感词汇
 gem 'harmonious_dictionary', '~> 0.0.1'
+
+gem 'parallel', '~> 1.19', '>= 1.19.1'

app/controllers/item_banks_controller.rb

@@ -43,9 +43,8 @@ class ItemBanksController < ApplicationController
       ApplyAction.where(container_type: "ItemBank", container_id: @item.id).destroy_all
       if @item.item_type == "PROGRAM"
         @item.container&.destroy!
-      else
-        @item.destroy!
       end
+      @item.destroy!
       render_ok
     end
   end

app/services/subjects/course_used_info_service.rb

@@ -21,17 +21,19 @@ class Subjects::CourseUsedInfoService < ApplicationService
     # choice_shixun_num: 选用该课程实训的个数（去重）
     # choice_shixun_frequency: 选用该课程实训的次数
     course_info = []
-    schools.map do |school|
+    schools.find_in_batches do |s|
-      name = school.name
+      Parallel.each(s) do |school|
-      course_count = school.course_count
+        name = school.name
-      student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size
+        course_count = school.course_count
-      shixun_ids = school.courses.joins(homework_commons: :homework_commons_shixun)
+        student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size
-                             .where(homework_commons: {id: homework_common_ids})
+        shixun_ids = school.courses.joins(homework_commons: :homework_commons_shixun)
-                             .pluck("homework_commons_shixuns.shixun_id")
+                         .where(homework_commons: {id: homework_common_ids})
-      choice_shixun_frequency = shixun_ids.size
+                         .pluck("homework_commons_shixuns.shixun_id")
-      choice_shixun_num = shixun_ids.uniq.size
+        choice_shixun_frequency = shixun_ids.size
-      course_info << {school_id: school.id, school_name: name, course_count: course_count, student_count: student_count,
+        choice_shixun_num = shixun_ids.uniq.size
-                      choice_shixun_num: choice_shixun_num, choice_shixun_frequency: choice_shixun_frequency}
+        course_info << {school_id: school.id, school_name: name, course_count: course_count, student_count: student_count,
+                        choice_shixun_num: choice_shixun_num, choice_shixun_frequency: choice_shixun_frequency}
+      end
     end
     course_info
   end

app/services/subjects/shixun_used_info_service.rb

@@ -10,20 +10,23 @@ class Subjects::ShixunUsedInfoService < ApplicationService
     stages.each do |stage|
       position = stage.position
       shixuns = stage.shixuns.includes(myshixuns: :games, homework_commons: :course)
-      shixuns.each_with_index do |shixun, index|
+      shixuns.find_in_batches(batch_size: 1000) do |s|
-        stage = "#{position}-#{index+1}"
+        Parallel.each_with_index(s, in_processes: 2) do |shixun, index|
-        name = shixun.name
+          stage = "#{position}-#{index+1}"
-        myshixuns = shixun.myshixuns
+          name = shixun.name
-        challenge_count = shixun.challenges_count
+          myshixuns = shixun.myshixuns
-        course_count = shixun.homework_commons.map{|hc| hc.course_id}.uniq.size
+          challenge_count = shixun.challenges_count
-        school_count = shixun.homework_commons.map{|hc| hc.course&.school_id}.uniq.size
+          course_count = shixun.homework_commons.map{|hc| hc.course_id}.uniq.size
-        used_count = shixun.myshixuns_count
+          school_count = shixun.homework_commons.map{|hc| hc.course&.school_id}.uniq.size
-        passed_count = myshixuns.select{|m| m.status == 1}.size
+          used_count = shixun.myshixuns_count
-        evaluate_count = myshixuns.map{|m| m.output_times }.sum
+          passed_count = myshixuns.select{|m| m.status == 1}.size
-        passed_ave_time = passed_count > 0 ? myshixuns.map{|m| m.total_cost_time}.sum : 0
+          evaluate_count = myshixuns.map{|m| m.output_times }.sum
-        shixun_infos << {stage: stage, name: name, challenge_count: challenge_count, course_count: course_count,
+          passed_ave_time = passed_count > 0 ? myshixuns.map{|m| m.total_cost_time}.sum : 0
-                         school_count: school_count, used_count: used_count, passed_count: passed_count,
+          shixun_infos << {stage: stage, name: name, challenge_count: challenge_count, course_count: course_count,
-                         evaluate_count: evaluate_count, passed_ave_time: passed_ave_time, shixun_id: shixun.id}
+                           school_count: school_count, used_count: used_count, passed_count: passed_count,
+                           evaluate_count: evaluate_count, passed_ave_time: passed_ave_time, shixun_id: shixun.id}
+
+        end
       end
     end
     shixun_infos

app/services/subjects/user_used_info_service.rb

@@ -1,6 +1,8 @@
 class Subjects::UserUsedInfoService < ApplicationService
+  require 'parallel'
   attr_reader :subject, :shixun_ids
 
+
   def initialize(subject)
     @subject = subject
     @shixun_ids = subject.shixuns.pluck(:id)
@@ -9,19 +11,22 @@ class Subjects::UserUsedInfoService < ApplicationService
 
   def call
     users_info = []
-    users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2})
+    users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2}, users: {is_test: false})
-    users.find_each do |user|
+    users.find_in_batches(batch_size: 500) do |u|
-      myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)}
+      Parallel.each(u, in_processes: 2) do |user|
-      name =  "#{user.lastname}#{user.firstname}"
+        myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)}
-      passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size
+        name =  "#{user.lastname}#{user.firstname}"
-      passed_games_count = myshixuns.map{|m| m.games.select{|g| g.status == 2}.size }.size
+        passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size
-      code_line_count = 0
+        passed_games_count = myshixuns.map{|m| m.games.select{|g| g.status == 2}.size }.size
-      evaluate_count = myshixuns.map{|m| m.output_times }.sum
+        code_line_count = 0
-      cost_time = myshixuns.map{|m|m.total_cost_time }.sum
+        evaluate_count = myshixuns.map{|m| m.output_times }.sum
-      users_info << {user_id: user.id, name: name, passed_myshixun_count: passed_myshixun_count,
+        cost_time = myshixuns.map{|m|m.total_cost_time }.sum
-                     passed_games_count: passed_games_count, code_line_count: code_line_count,
+        users_info << {user_id: user.id, name: name, passed_myshixun_count: passed_myshixun_count,
-                    evaluate_count: evaluate_count, cost_time: cost_time}
+                       passed_games_count: passed_games_count, code_line_count: code_line_count,
+                       evaluate_count: evaluate_count, cost_time: cost_time}
+      end
     end
+
     users_info
   end
 

lib/tasks/statistic_subject_info.rake

@@ -4,30 +4,32 @@ namespace :subjects do
   task data_statistic: :environment do
     puts("---------------------data_statistic_begin")
     Rails.logger.info("---------------------data_statistic_begin")
-    subjects = Subject.where(status: 2)
+    subjects = Subject.where(status: 2, hidden: 0)
     str = ""
     buffer_size = 0
     column_value = "subject_id, study_count, course_study_count, initiative_study, passed_count, course_used_count, " +
         "school_used_count, created_at, updated_at"
-    subjects.find_each do |subject|
+    subjects.find_in_batches(batch_size: 50) do |s|
-      puts("---------------------data_statistic: #{subject.id}")
+      Parallel.each(s, in_processes: 4) do |subject|
-      Rails.logger.info("---------------------data_statistic: #{subject.id}")
+        puts("---------------------data_statistic: #{subject.id}")
-      data = Subjects::DataStatisticService.new(subject)
+        Rails.logger.info("---------------------data_statistic: #{subject.id}")
-      study_count = data.study_count
+        data = Subjects::DataStatisticService.new(subject)
-      next if study_count == 0
+        study_count = data.study_count
-      course_study_count = data.course_study_count
+        next if study_count == 0
-      initiative_study = study_count - course_study_count
+        course_study_count = data.course_study_count
-      str += ", " unless str.empty?
+        initiative_study = study_count - course_study_count
-      str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " +
+        str += ", " unless str.empty?
-          "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " +
+        str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " +
-          "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
+            "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " +
-      buffer_size += 1
+            "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-      if buffer_size == 1000
+        buffer_size += 1
-        sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}"
+        if buffer_size == 1000
-        puts sql
+          sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}"
-        ActiveRecord::Base.connection.execute sql
+          puts sql
-        str = ""
+          ActiveRecord::Base.connection.execute sql
-        buffer_size = 0
+          str = ""
+          buffer_size = 0
+        end
       end
     end
     if buffer_size > 0
@@ -42,28 +44,31 @@ namespace :subjects do
   task course_info_statistic: :environment do
     puts("---------------------course_info_statistic_begin")
     Rails.logger.info("---------------------course_info_statistic_begin")
-    subjects = Subject.where(status: 2)
+    subjects = Subject.where(status: 2, hidden: 0)
     str = ""
     buffer_size = 0
     column_value = "subject_id, school_id, school_name, course_count, student_count, choice_shixun_num, " +
         "choice_shixun_frequency, created_at, updated_at"
-    subjects.find_each do |subject|
+
-      puts("---------------------course_info_statistic: #{subject.id}")
+    subjects.find_in_batches(batch_size: 50) do |s|
-      Rails.logger.info("---------------------course_info_statistic: #{subject.id}")
+      Parallel.each(s, in_processes: 4) do |subject|
-      data = Subjects::CourseUsedInfoService.call(subject)
+        puts("---------------------course_info_statistic: #{subject.id}")
-      data.each do |key|
+        Rails.logger.info("---------------------course_info_statistic: #{subject.id}")
-        next if key[:school_id].nil?
+        data = Subjects::CourseUsedInfoService.call(subject)
-        str += ", " unless str.empty?
+        Parallel.map(data) do |key|
-        str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " +
+          next if key[:school_id].nil?
-            "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " +
+          str += ", " unless str.empty?
-            "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
+          str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " +
-        buffer_size += 1
+              "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " +
-        if buffer_size == 1000
+              "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-          sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}"
+          buffer_size += 1
-          puts sql
+          if buffer_size == 1000
-          ActiveRecord::Base.connection.execute sql
+            sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}"
-          str = ""
+            puts sql
-          buffer_size = 0
+            ActiveRecord::Base.connection.execute sql
+            str = ""
+            buffer_size = 0
+          end
         end
       end
     end
@@ -79,29 +84,31 @@ namespace :subjects do
   task shixun_info_statistic: :environment do
     puts("---------------------shixun_info_statistic_begin")
     Rails.logger.info("---------------------shixun_info_statistic_begin")
-    subjects = Subject.where(status: 2)
+    subjects = Subject.where(status: 2, hidden: 0)
     str = ""
     buffer_size = 0
     column_value = "subject_id, shixun_id, stage, shixun_name, challenge_count, course_count, " +
         "school_count, used_count, passed_count, evaluate_count, passed_ave_time, created_at, updated_at"
-    subjects.find_each(batch_size: 100) do |subject|
+    subjects.find_in_batches(batch_size: 50) do |s|
-      puts("---------------------shixun_info_statistic: #{subject.id}")
+      Parallel.each(s, in_processes: 4) do |subject|
-      Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}")
+        puts("---------------------shixun_info_statistic: #{subject.id}")
-      data = Subjects::ShixunUsedInfoService.call(subject)
+        Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}")
-      data.each do |key|
+        data = Subjects::ShixunUsedInfoService.call(subject)
-        next if key[:shixun_id].nil?
+        data.each do |key|
-        str += ", " unless str.empty?
+          next if key[:shixun_id].nil?
-        str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " +
+          str += ", " unless str.empty?
-            "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " +
+          str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " +
-            "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " +
+              "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " +
-            "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
+              "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " +
-        buffer_size += 1
+              "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-        if buffer_size == 1000
+          buffer_size += 1
-          sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}"
+          if buffer_size == 1000
-          puts sql
+            sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}"
-          ActiveRecord::Base.connection.execute sql
+            puts sql
-          str = ""
+            ActiveRecord::Base.connection.execute sql
-          buffer_size = 0
+            str = ""
+            buffer_size = 0
+          end
         end
       end
     end
@@ -117,26 +124,29 @@ namespace :subjects do
   task user_info_statistic: :environment do
     puts("---------------------user_info_statistic_begin")
     Rails.logger.info("---------------------user_info_statistic_begin")
-    subjects = Subject.where(status: 2)
+    subjects = Subject.where(status: 2, hidden: 0)
     str = ""
     buffer_size = 0
     column_value = "user_id, subject_id, username, passed_myshixun_count, passed_games_count, " +
         "code_line_count, evaluate_count, cost_time, created_at, updated_at"
-    subjects.find_each(batch_size: 50) do |subject|
+
-      puts("---------------------user_info_statistic: #{subject.id}")
+    subjects.find_in_batches(batch_size: 50) do |s|
-      data = Subjects::UserUsedInfoService.call(subject)
+      Parallel.each(s, in_processes: 4) do |subject|
-      data.each do |key|
+        puts("---------------------user_info_statistic: #{subject.id}")
-        next if key[:user_id].nil?
+        data = Subjects::UserUsedInfoService.call(subject)
-        str += ", " unless str.empty?
+        data.each do |key|
-        str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name]}', '#{key[:passed_myshixun_count]}', " +
+          next if key[:user_id].nil?
-            "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " +
+          str += ", " unless str.empty?
-            "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
+          str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " +
-        buffer_size += 1
+              "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " +
-        if buffer_size == 1000
+              "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-          sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}"
+          buffer_size += 1
-          ActiveRecord::Base.connection.execute sql
+          if buffer_size == 1000
-          str = ""
+            sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}"
-          buffer_size = 0
+            ActiveRecord::Base.connection.execute sql
+            str = ""
+            buffer_size = 0
+          end
         end
       end
     end