Merge branches 'dev_aliyun' and 'develop' of https://bdgit.educoder.net/Hjqreturn/educoder into dev_aliyun

app/controllers/myshixuns_controller.rb

@@ -265,6 +265,7 @@ class MyshixunsController < ApplicationController
         uid_logger_dubug("-- game build: file update #{@sec_key}, record id is #{record.id}, time is **** #{Time.now.strftime("%Y-%m-%d %H:%M:%S.%L")}")
       end
       # 隐藏代码文件 和  VNC的都不需要走版本库
+      uid_logger_dubug("#######enter_#{@myshixun.identifier}: #{@hide_code}")
       unless @hide_code || (@myshixun.shixun&.vnc_evaluate && params[:evaluate].present?)
         # 远程版本库文件内容
         last_content = GitService.file_content(repo_path: @repo_path, path: path)["content"]
@@ -275,8 +276,8 @@ class MyshixunsController < ApplicationController
             else
               params[:content]
             end
-        uid_logger_dubug("###11222333####{content}")
+        uid_logger_dubug("content_#{@myshixun.identifier}: #{content}")
-        uid_logger_dubug("###222333####{last_content}")
+        uid_logger_dubug("###last_content_#{@myshixun.identifier}####{last_content}")
 
         if content != last_content
           @content_modified = 1
@@ -286,6 +287,7 @@ class MyshixunsController < ApplicationController
           message = params[:evaluate] == 0 ? "System automatically submitted" : "User submitted"
           uid_logger_dubug("112233#{author_name}")
           uid_logger_dubug("112233#{author_email}")
+          uid_logger_dubug("daiao_debug_#{@myshixun.identifier}: #{@repo_path}: #{path}; #{message}; #{content}; ")
           @content = GitService.update_file(repo_path: @repo_path,
                                             file_path: path,
                                             message: message,

app/services/subjects/course_used_info_service.rb

@@ -22,7 +22,7 @@ class Subjects::CourseUsedInfoService < ApplicationService
     # choice_shixun_frequency: 选用该课程实训的次数
     course_info = []
     schools.find_in_batches do |s|
-      Parallel.each(s) do |school|
+      s.each do |school|
         name = school.name
         course_count = school.course_count
         student_count = school.courses.joins(:course_members).where(course_members: {role: 4, course_id: course_ids}).size

app/services/subjects/shixun_used_info_service.rb

@@ -11,7 +11,7 @@ class Subjects::ShixunUsedInfoService < ApplicationService
       position = stage.position
       shixuns = stage.shixuns.includes(myshixuns: :games, homework_commons: :course)
       shixuns.find_in_batches(batch_size: 1000) do |s|
-        Parallel.each_with_index(s, in_processes: 2) do |shixun, index|
+        s.each_with_index do |shixun, index|
           stage = "#{position}-#{index+1}"
           name = shixun.name
           myshixuns = shixun.myshixuns

app/services/subjects/user_used_info_service.rb

@@ -13,7 +13,7 @@ class Subjects::UserUsedInfoService < ApplicationService
     users_info = []
     users = User.includes(myshixuns: :games).where(myshixuns: {shixun_id: shixun_ids}, games: {status: 2}, users: {is_test: false})
     users.find_in_batches(batch_size: 500) do |u|
-      Parallel.each(u, in_processes: 2) do |user|
+      u.each do |user|
         myshixuns = user.myshixuns.select{|m| shixun_ids.include?(m.shixun_id)}
         name =  "#{user.lastname}#{user.firstname}"
         passed_myshixun_count = myshixuns.select{|m| m.status == 1}.size

lib/tasks/statistic_subject_info.rake

@@ -5,14 +5,14 @@ namespace :subjects do
     puts("---------------------data_statistic_begin")
     Rails.logger.info("---------------------data_statistic_begin")
     subjects = Subject.where(status: 2, hidden: 0)
+    if ENV['subject_id'].present?
+      subjects = subjects.where(id:ENV['subject_id'])
+    end
     column_value = "subject_id, study_count, course_study_count, initiative_study, passed_count, course_used_count, " +
         "school_used_count, created_at, updated_at"
     subjects.find_in_batches(batch_size: 50) do |s|
-      str = ""
+      str = []
-      buffer_size = 0
+      Parallel.each_with_index(s, in_threads: 4) do |subject, index|
-      Parallel.each(s, in_processes: 4) do |subject|
-        buffer_size += 1
-        str += ", " unless str.empty?
         puts("---------------------data_statistic: #{subject.id}")
         Rails.logger.info("---------------------data_statistic: #{subject.id}")
         data = Subjects::DataStatisticService.new(subject)
@@ -20,25 +20,19 @@ namespace :subjects do
         next if study_count == 0
         course_study_count = data.course_study_count
         initiative_study = study_count - course_study_count
-        str += ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " +
+        str << ("(#{subject.id}, #{study_count}, #{course_study_count}, #{initiative_study}, " +
             "#{data.passed_count}, #{data.course_used_count}, #{data.school_used_count}, " +
             "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-        puts "buffer_size: #{buffer_size}; subject_id: #{subject.id}; s: #{s.last}; subject: #{subject == s.last};"
+        puts "index: #{index}; worker_number: #{Parallel.worker_number}"
         puts "####str: #{str}"
-        if buffer_size == 1000 || subject == s.last
+      end
-          sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}"
+      if str.size > 0
-          puts sql
+        sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str.uniq.join(",")}"
-          ActiveRecord::Base.connection.execute sql
+        puts sql
-          str = ""
+        ActiveRecord::Base.connection.execute sql
-          buffer_size = 0
-        end
       end
     end
-    if buffer_size > 0
+
-      sql = "REPLACE INTO subject_records(#{column_value}) VALUES #{str}"
-      puts sql
-      ActiveRecord::Base.connection.execute sql
-    end
     puts("---------------------data_statistic_end")
     Rails.logger.info("---------------------data_statistic_end")
   end
@@ -47,38 +41,37 @@ namespace :subjects do
     puts("---------------------course_info_statistic_begin")
     Rails.logger.info("---------------------course_info_statistic_begin")
     subjects = Subject.where(status: 2, hidden: 0)
-    str = ""
-    buffer_size = 0
     column_value = "subject_id, school_id, school_name, course_count, student_count, choice_shixun_num, " +
         "choice_shixun_frequency, created_at, updated_at"
-
+    if ENV['subject_id'].present?
-    subjects.find_in_batches(batch_size: 50) do |s|
+      subjects = subjects.where(id:ENV['subject_id'])
-      Parallel.each(s, in_processes: 4) do |subject|
+    end
+    subjects.find_in_batches(batch_size: 20) do |s|
+      str = []
+      Parallel.each(s, in_processes: 6) do |subject|
         puts("---------------------course_info_statistic: #{subject.id}")
         Rails.logger.info("---------------------course_info_statistic: #{subject.id}")
         data = Subjects::CourseUsedInfoService.call(subject)
         data.each do |key|
           next if key[:school_id].nil?
-          str += ", " unless str.empty?
+          str << ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " +
-          str += ("(#{subject.id}, #{key[:school_id]}, '#{key[:school_name]}', #{key[:course_count]}, " +
               "#{key[:student_count]}, #{key[:choice_shixun_num]}, #{key[:choice_shixun_frequency]}, " +
               "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-          buffer_size += 1
+          # if str.size == 1000
-          if buffer_size == 1000 || key == data.last
+          #   sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str.uniq.join(",")}"
-            sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}"
+          #   str_c = str
-            puts sql
+          #   puts sql
-            ActiveRecord::Base.connection.execute sql
+          #   ActiveRecord::Base.connection.execute sql
-            str = ""
+          #   str -= str_c
-            buffer_size = 0
+          # end
-          end
+        end
+        if str.size > 0
+          sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str.uniq.join(",")}"
+          puts sql
+          ActiveRecord::Base.connection.execute sql
         end
       end
     end
-    if buffer_size > 0
-      sql = "REPLACE INTO subject_course_records(#{column_value}) VALUES #{str}"
-      puts sql
-      ActiveRecord::Base.connection.execute sql
-    end
     puts("---------------------course_info_statistic_end")
     Rails.logger.info("---------------------course_info_statistic_end")
   end
@@ -87,38 +80,35 @@ namespace :subjects do
     puts("---------------------shixun_info_statistic_begin")
     Rails.logger.info("---------------------shixun_info_statistic_begin")
     subjects = Subject.where(status: 2, hidden: 0)
-    str = ""
+    if ENV['subject_id'].present?
-    buffer_size = 0
+      subjects = subjects.where(id:ENV['subject_id'])
+    end
     column_value = "subject_id, shixun_id, stage, shixun_name, challenge_count, course_count, " +
         "school_count, used_count, passed_count, evaluate_count, passed_ave_time, created_at, updated_at"
-    subjects.find_in_batches(batch_size: 50) do |s|
+    subjects.find_in_batches(batch_size: 20) do |s|
-      Parallel.each_with_index(s, in_processes: 4) do |subject|
+      Parallel.each_with_index(s, in_processes: 4) do |subject, index, str = []|
         puts("---------------------shixun_info_statistic: #{subject.id}")
         Rails.logger.info("---------------------shixun_info_statistic: #{subject.id}")
         data = Subjects::ShixunUsedInfoService.call(subject)
         data.each do |key|
           next if key[:shixun_id].nil?
-          str += ", " unless str.empty?
+          str << ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " +
-          str += ("(#{subject.id}, #{key[:shixun_id]}, '#{key[:stage]}', '#{key[:name]}', #{key[:challenge_count]}, " +
               "#{key[:course_count]}, #{key[:school_count]}, #{key[:used_count]}, #{key[:passed_count]}, " +
               "#{key[:evaluate_count]}, #{key[:passed_ave_time]}, " +
               "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-          buffer_size += 1
+          # if str.size == 1000
-          if buffer_size == 1000 || key == data.last
+          #   sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str.join(",")}"
-            sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}"
+          #   puts sql
-            puts sql
+          #   ActiveRecord::Base.connection.execute sql
-            ActiveRecord::Base.connection.execute sql
+          # end
-            str = ""
+        end
-            buffer_size = 0
+        if str.size > 0
-          end
+          sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str.join(",")}"
+          puts sql
+          ActiveRecord::Base.connection.execute sql
         end
       end
     end
-    if buffer_size > 0
-      sql = "REPLACE INTO subject_shixun_infos(#{column_value}) VALUES #{str}"
-      puts sql
-      ActiveRecord::Base.connection.execute sql
-    end
     puts("---------------------shixun_info_statistic_end")
     Rails.logger.info("---------------------shixun_info_statistic_end")
   end
@@ -127,36 +117,29 @@ namespace :subjects do
     puts("---------------------user_info_statistic_begin")
     Rails.logger.info("---------------------user_info_statistic_begin")
     subjects = Subject.where(status: 2, hidden: 0)
-    str = ""
-    buffer_size = 0
     column_value = "user_id, subject_id, username, passed_myshixun_count, passed_games_count, " +
         "code_line_count, evaluate_count, cost_time, created_at, updated_at"
-
+    subjects.find_in_batches(batch_size: 20) do |s|
-    subjects.find_in_batches(batch_size: 50) do |s|
+      Parallel.each_with_index(s, in_processes: 4) do |subject, index, str = []|
-      Parallel.each_with_index(s, in_processes: 4) do |subject, index|
         puts("---------------------user_info_statistic: #{subject.id}")
         data = Subjects::UserUsedInfoService.call(subject)
         data.each do |key|
-          next if key[:user_id].nil?
+          str << ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " +
-          str += ", " unless str.empty?
-          str += ("(#{key[:user_id]}, #{subject.id}, '#{key[:name].gsub(/'/, '"')}', #{key[:passed_myshixun_count]}, " +
               "#{key[:passed_games_count]}, #{key[:code_line_count]}, #{key[:evaluate_count]}, #{key[:cost_time]}, " +
               "'#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}', '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}')")
-          buffer_size += 1
+          # if str.size == 1000
-          if buffer_size == 1000 || (key == data.last)
+          #   sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str.join(",")}"
-            sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}"
+          #   ActiveRecord::Base.connection.execute sql
-            ActiveRecord::Base.connection.execute sql
+          #   str = []
-            str = ""
+          # end
-            buffer_size = 0
+        end
-          end
+        if str.size > 0
+          sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str.join(",")}"
+          puts sql
+          ActiveRecord::Base.connection.execute sql
         end
       end
     end
-    if buffer_size > 0
-      sql = "REPLACE INTO subject_user_infos(#{column_value}) VALUES #{str}"
-      puts sql
-      ActiveRecord::Base.connection.execute sql
-    end
     puts("---------------------user_info_statistic_end")
     Rails.logger.info("---------------------user_info_statistic_end")
   end