ES search feature

Gemfile

@@ -89,3 +89,5 @@ gem 'sinatra'
 # batch insert
 gem 'bulk_insert'
 
+# elasticsearch
+gem 'searchkick'

Gemfile.lock

@@ -90,6 +90,14 @@ GEM
     connection_pool (2.2.2)
     crass (1.0.4)
     diff-lcs (1.3)
+    elasticsearch (7.1.0)
+      elasticsearch-api (= 7.1.0)
+      elasticsearch-transport (= 7.1.0)
+    elasticsearch-api (7.1.0)
+      multi_json
+    elasticsearch-transport (7.1.0)
+      faraday
+      multi_json
     erubi (1.7.1)
     execjs (2.7.0)
     faraday (0.15.4)
@@ -100,6 +108,7 @@ GEM
     grape-entity (0.7.1)
       activesupport (>= 4.0)
       multi_json (>= 1.3.2)
+    hashie (3.6.0)
     htmlentities (4.3.4)
     httparty (0.16.2)
       multi_xml (>= 0.5.2)
@@ -255,6 +264,10 @@ GEM
       sprockets (>= 2.8, < 4.0)
       sprockets-rails (>= 2.0, < 4.0)
       tilt (>= 1.1, < 3)
+    searchkick (3.1.3)
+      activemodel (>= 4.2)
+      elasticsearch (>= 5)
+      hashie
     selenium-webdriver (3.14.0)
       childprocess (~> 0.5)
       rubyzip (~> 1.2)
@@ -344,6 +357,7 @@ DEPENDENCIES
   ruby-ole
   rubyzip
   sass-rails (~> 5.0)
+  searchkick
   selenium-webdriver
   sidekiq
   simple_xlsx_reader

app/controllers/homework_commons_controller.rb

@@ -410,10 +410,8 @@ class HomeworkCommonsController < ApplicationController
           homework_detail_group = @homework.homework_detail_group
           param_min = params[:min_num].to_i
           param_max = params[:max_num].to_i
-          homework_detail_group.min_num = @homework.has_commit_work ? (param_min > homework_detail_group.min_num ? homework_detail_group.min_num :
+          homework_detail_group.min_num = @homework.has_commit_work ? [param_min, homework_detail_group.min_num].min : param_min
-                                                                           param_min) : param_min
+          homework_detail_group.max_num = @homework.has_commit_work ? [param_max, homework_detail_group.max_num].max : param_max
-          homework_detail_group.max_num = @homework.has_commit_work ? (param_max < homework_detail_group.max_num ? homework_detail_group.max_num :
-                                                                           param_max) : param_max
           homework_detail_group.base_on_project = params[:base_on_project] unless @homework.has_relate_project
           homework_detail_group.save!
         end

app/controllers/searchs_controller.rb

@@ -0,0 +1,10 @@
+class SearchsController < ApplicationController
+  def index
+    @results = SearchService.call(search_params)
+  end
+
+  private
+  def search_params
+    params.permit(:keyword, :type, :page, :per_page)
+  end
+end

app/controllers/shixuns_controller.rb

@@ -1,4 +1,7 @@
 class ShixunsController < ApplicationController
+	include ShixunsHelper
+	include ApplicationHelper
+
 	before_action :require_login, :check_auth, except: [:download_file, :index, :menus]
 	before_action :check_auth, except: [:download_file, :index, :menus]
 
@@ -14,9 +17,6 @@ class ShixunsController < ApplicationController
 
 	before_action :special_allowed, only: [:send_to_course, :search_user_courses]
 
-	include ShixunsHelper
-	include ApplicationHelper
-
 	## 获取课程列表
   def index
     ## 我的实训
@@ -84,6 +84,12 @@ class ShixunsController < ApplicationController
 		limit = params[:limit] || 16
 
 		@shixuns = @shixuns.includes(:tag_repertoires, :challenges).page(page).per(limit)
+
+		@tag_name_map = TagRepertoire.joins(:shixun_tag_repertoires)
+											.where(shixun_tag_repertoires: { shixun_id: @shixuns.map(&:id) })
+											.group('shixun_tag_repertoires.shixun_id')
+											.select('shixun_id, tag_repertoires.name')
+											.each_with_object({}) { |r, obj| obj[r.shixun_id] = r.name }
   end
 
   ## 获取顶部菜单

app/libs/util.rb

@@ -33,4 +33,12 @@ module Util
     Rails.logger.error(exception.message)
     exception.backtrace.each { |message| Rails.logger.error(message) }
   end
+
+  def map_or_pluck(relation, name)
+    relation.is_a?(Array) || relation.loaded? ? relation.map(&name.to_sym) : relation.pluck(name)
+  end
+
+  def extract_content(str)
+    str.gsub(/<\/?.*?>/, '').gsub(/[\n\t\r]/, '').gsub(/&nbsp;/, '')
+  end
 end

app/models/challenge.rb

@@ -116,5 +116,4 @@ class Challenge < ApplicationRecord
   end
 
   # 关卡评测文件
-
 end

app/models/challenge_tag.rb

@@ -1,4 +1,6 @@
 class ChallengeTag < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Dependents::ChallengeTag
 
   belongs_to :challenge, counter_cache: true
   belongs_to :challenge_choose, optional: true

app/models/course.rb

@@ -1,4 +1,7 @@
 class Course < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Course
+
   has_many :boards, dependent: :destroy
 
   belongs_to :teacher, class_name: 'User', foreign_key: :tea_id  # 定义一个方法teacher，该方法通过tea_id来调用User表
@@ -22,6 +25,8 @@ class Course < ApplicationRecord
   has_many :graduation_groups, dependent: :destroy
 
   has_many :course_members, dependent: :destroy
+  has_many :teacher_course_members, -> { teachers_and_admin }, class_name: 'CourseMember'
+  has_many :teacher_users, through: :teacher_course_members, source: :user
   has_many :course_messages, dependent: :destroy
   has_many :homework_commons, dependent: :destroy
   has_many :homework_group_settings

app/models/memo.rb

@@ -1,4 +1,6 @@
 class Memo < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Memo
 
   has_many :memo_tag_repertoires, :dependent => :destroy
   has_many :tag_repertoires, :through => :memo_tag_repertoires
@@ -9,6 +11,9 @@ class Memo < ApplicationRecord
   belongs_to :author, class_name: 'User', foreign_key: 'author_id'
   belongs_to :parent, class_name: 'Memo', foreign_key: 'parent_id'
 
+  has_many :descendants, foreign_key: :root_id, class_name: 'Memo'
+  has_many :children, foreign_key: :parent_id, class_name: 'Memo'
+
   scope :field_for_list, lambda{
     select([:id, :subject, :author_id, :sticky, :updated_at, :language, :reward, :all_replies_count, :viewed_count, :forum_id])
   }

app/models/searchable.rb

@@ -0,0 +1,3 @@
+module Searchable
+  MAXIMUM_LENGTH = 10922 # 最大字节数为32766 ，一个汉字3个字节
+end

app/models/searchable/course.rb

@@ -0,0 +1,37 @@
+module Searchable::Course
+  extend ActiveSupport::Concern
+
+  included do
+    searchkick language: 'chinese', callbacks: :async
+
+    scope :search_import, -> { includes(:teacher_users, teacher: { user_extension: :school } ) }
+  end
+
+  def searchable_title
+    name
+  end
+
+  def search_data
+    {
+      name: name,
+      author_name: teacher&.real_name
+    }
+  end
+
+  def to_searchable_json
+    {
+      id: id,
+      author_name: teacher.real_name,
+      author_school_name: teacher.school_name,
+      visits_count: visits,
+      members_count: members_count,
+      is_public: is_public == 1
+    }
+  end
+
+  module ClassMethods
+    def searchable_includes
+      { teacher: { user_extension: :school } }
+    end
+  end
+end

app/models/searchable/dependents.rb

@@ -0,0 +1,2 @@
+module Searchable::Dependents
+end

app/models/searchable/dependents/challenge_tag.rb

@@ -0,0 +1,16 @@
+module Searchable::Dependents::ChallengeTag
+  extend ActiveSupport::Concern
+
+  included do
+    after_create_commit :check_searchable_dependents
+    after_update_commit :check_searchable_dependents
+  end
+
+  private
+
+  def check_searchable_dependents
+    if new_record? || name_previously_changed?
+      challenge.shixun.reindex(:searchable_challenge_data)
+    end
+  end
+end

app/models/searchable/dependents/stage.rb

@@ -0,0 +1,15 @@
+module Searchable::Dependents::Stage
+  extend ActiveSupport::Concern
+
+  included do
+    after_update_commit :check_searchable_dependents
+  end
+
+  private
+
+  def check_searchable_dependents
+    if name_previously_changed? || description_previously_changed?
+      subject.reindex(:searchable_stages_data)
+    end
+  end
+end

app/models/searchable/dependents/user.rb

@@ -0,0 +1,22 @@
+module Searchable::Dependents::User
+  extend ActiveSupport::Concern
+
+  included do
+    after_update_commit :check_searchable_dependents
+  end
+
+  private
+
+  def check_searchable_dependents
+    if firstname_previously_changed? || lastname_previously_changed? || user_extension.school_id_previously_changed?
+      # reindex shixun
+      created_shixuns.each{ |shixun| shixun.reindex(:searchable_user_data) }
+
+      # reindex course
+      manage_courses.each(&:reindex)
+
+      # reindex subject
+      created_subjects.each { |subject| subject.reindex(:searchable_user_data) }
+    end
+  end
+end

app/models/searchable/memo.rb

@@ -0,0 +1,46 @@
+module Searchable::Memo
+  extend ActiveSupport::Concern
+
+  included do
+    searchkick language: 'chinese', callbacks: :async
+
+    scope :search_import, -> { includes(:descendants) }
+  end
+
+  def searchable_title
+    subject
+  end
+
+  def should_index?
+    hidden.zero? && root_id.blank? && parent_id.blank?
+  end
+
+  def search_data
+    {
+      name: subject,
+      content: Util.extract_content(content)[0..Searchable::MAXIMUM_LENGTH],
+    }.merge!(searchable_descendants_data)
+  end
+
+  def searchable_descendants_data
+    {
+      descendants_contents: Util.map_or_pluck(descendants, :content)
+                              .map { |content| Util.extract_content(content)[0..Searchable::MAXIMUM_LENGTH] }
+    }
+  end
+
+  def to_searchable_json
+    {
+      id: id,
+      author_name: author.full_name,
+      visits_count: viewed_count,
+      all_replies_count: all_replies_count
+    }
+  end
+
+  module ClassMethods
+    def searchable_includes
+      [:author]
+    end
+  end
+end

app/models/searchable/shixun.rb

@@ -0,0 +1,58 @@
+module Searchable::Shixun
+  extend ActiveSupport::Concern
+
+  included do
+    searchkick language: 'chinese'#, callbacks: :async
+
+    scope :search_import, -> { includes(:shixun_info, :challenges, :challenge_tags, :users, user: { user_extension: :school }) }
+  end
+
+  def searchable_title
+    name
+  end
+
+  def search_data
+    {
+      name: name,
+      description: Util.extract_content(description)[0..Searchable::MAXIMUM_LENGTH]
+    }.merge!(searchable_user_data)
+      .merge!(searchable_challenge_data)
+  end
+
+  def searchable_user_data
+    {
+      author_name: user.real_name,
+      author_school_name: user.school_name,
+    }
+  end
+
+  def searchable_challenge_data
+    challenge_names = Util.map_or_pluck(challenges, :subject)
+                        .each_with_index.map { |subject, index| "第#{index + 1}关 #{subject}" }
+
+    {
+      challenge_names: challenge_names,
+      challenge_tag_names: Util.map_or_pluck(challenge_tags, :name).uniq.join(' ')
+    }
+  end
+
+  def should_index?
+    status == 2 # published
+  end
+
+  def to_searchable_json
+    {
+      id: id,
+      author_name: user.real_name,
+      author_school_name: user.school_name,
+      visits_count: visits,
+      challenges_count: challenges_count
+    }
+  end
+
+  module ClassMethods
+    def searchable_includes
+      { user: { user_extension: :school } }
+    end
+  end
+end

app/models/searchable/subject.rb

@@ -0,0 +1,61 @@
+module Searchable::Subject
+  extend ActiveSupport::Concern
+
+  included do
+    searchkick language: 'chinese', callbacks: :async
+
+    scope :search_import, -> { includes(:users, :stages, user: { user_extension: :school }) }
+  end
+
+  def searchable_title
+    name
+  end
+
+  def should_index?
+    !hidden? && status == 2 # published
+  end
+
+  def search_data
+    {
+      name: name,
+      description: Util.extract_content(description)[0..Searchable::MAXIMUM_LENGTH]
+    }.merge!(searchable_user_data)
+      .merge!(searchable_stages_data)
+  end
+
+  def searchable_user_data
+    {
+      author_name: user.real_name,
+      author_school_name: user.school_name,
+    }
+  end
+
+  def searchable_stages_data
+    subject_stages =
+      stages.map do |stage|
+        {
+          name: stage.name,
+          description: Util.extract_content(stage.description)[0..Searchable::MAXIMUM_LENGTH]
+        }
+      end
+
+    { subject_stages: subject_stages}
+  end
+
+  def to_searchable_json
+    {
+      id: id,
+      author_name: user.real_name,
+      author_school_name: user.school_name,
+      visits_count: visits,
+      stage_count: stages_count,
+      stage_shixuns_count: stage_shixuns_count
+    }
+  end
+
+  module ClassMethods
+    def searchable_includes
+      { user: { user_extension: :school } }
+    end
+  end
+end

app/models/shixun.rb

@@ -1,8 +1,12 @@
 class Shixun < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Shixun
+
   # status: 0：编辑   1：申请发布   2：正式发布  3：关闭  -1：软删除
   # hide_code： 隐藏代码窗口
   # code_hidden: 隐藏代码目录
   has_many :challenges, dependent: :destroy
+  has_many :challenge_tags, through: :challenges
 	has_many :myshixuns, :dependent => :destroy
   has_many :shixun_members, dependent: :destroy
   has_many :users, through: :shixun_members
@@ -35,7 +39,6 @@ class Shixun < ApplicationRecord
   # 实训服务配置
   has_many :shixun_service_configs, :dependent => :destroy
 
-
 	scope :search_by_name, ->(keyword) { where("name like ? or description like ? ",
 				"%#{keyword}%", "%#{keyword}%") }
 

app/models/stage.rb

@@ -1,4 +1,7 @@
 class Stage < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Dependents::Stage
+
   belongs_to :subject, counter_cache: true
 
   has_many :stage_shixuns, -> { order("stage_shixuns.position ASC") }, dependent: :destroy

app/models/subject.rb

@@ -2,6 +2,9 @@
 # 可以在初始创建的时候
 
 class Subject < ApplicationRecord
+  # TODO: ES feature
+  # include Searchable::Subject
+
   #status :0 编辑中  1 审核中  2 发布
   belongs_to :repertoire
   belongs_to :user

app/models/user.rb

@@ -1,5 +1,8 @@
 class User < ApplicationRecord
   include Watchable
+  # TODO: ES feature
+  # include Searchable::Dependents::User
+
   # Account statuses
   STATUS_ANONYMOUS  = 0
   STATUS_ACTIVE     = 1
@@ -28,6 +31,7 @@ class User < ApplicationRecord
   accepts_nested_attributes_for :user_extension, update_only: true
 
   has_many :memos, foreign_key: 'author_id'
+  has_many :created_shixuns, class_name: 'Shixun'
   has_many :shixun_members, :dependent => :destroy
   has_many :shixuns, :through => :shixun_members
   has_many :myshixuns, :dependent => :destroy
@@ -55,6 +59,7 @@ class User < ApplicationRecord
   has_many :tidings, :dependent => :destroy
 
   has_many :games, :dependent => :destroy
+  has_many :created_subjects
   has_many :subjects, :through => :subject_members
   has_many :subject_members, :dependent => :destroy
   has_many :grades, :dependent => :destroy

app/models/user_extension.rb

@@ -2,7 +2,7 @@ class UserExtension < ApplicationRecord
   # identity 0: 教师教授 1: 学生, 2: 专业人士, 3: 开发者
   enum identity: { teacher: 0, student: 1, professional: 2, developer: 3 }
 
-  belongs_to :user
+  belongs_to :user, touch: true
   belongs_to :school
   belongs_to :department, optional: true
 

app/services/concerns/elasticsearch_able.rb

@@ -0,0 +1,41 @@
+module ElasticsearchAble
+  extend ActiveSupport::Concern
+
+  private
+
+  def default_options
+    {
+      debug: Rails.env.development?,
+      highlight: highlight_options,
+      body_options: body_options,
+      page: page,
+      per_page: per_page
+    }
+  end
+
+  def keyword
+    params[:keyword].to_s.strip.presence || '*'
+  end
+
+  def highlight_options
+    {
+      fragment_size: EduSetting.get('es_highlight_fragment_size') || 30,
+      tag: '<span>'
+    }
+  end
+
+  def body_options
+    {
+      min_score: EduSetting.get('es_min_score') || 10
+    }
+  end
+
+  def per_page
+    per_page = params[:per_page].to_s.strip.presence || params[:limit].to_s.strip.presence
+    per_page.to_i <= 0 ? 20 : per_page.to_i
+  end
+
+  def page
+    params[:page].to_i <= 0 ? 1 : params[:page].to_i
+  end
+end

app/services/search_service.rb

@@ -0,0 +1,39 @@
+class SearchService < ApplicationService
+  include ElasticsearchAble
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def call
+    Searchkick.search(keyword, search_options)
+  end
+
+  private
+
+  def search_options
+    {
+      index_name: index_names,
+      model_includes: model_includes
+    }.merge(default_options)
+  end
+
+  def index_names
+    @_index_names ||=
+      case params[:type].to_s.strip
+      when 'shixun'  then [Shixun]
+      when 'course'  then [Course]
+      when 'subject' then [Subject]
+      when 'memo'    then [Memo]
+      else [Shixun, Course, Subject, Memo]
+      end
+  end
+
+  def model_includes
+    index_names.each_with_object({}) do |klass, obj|
+      obj[klass] = klass.searchable_includes
+    end
+  end
+end

app/services/search_shixun_service.rb

@@ -0,0 +1,96 @@
+class SearchShixunService < ApplicationService
+  include ElasticsearchAble
+
+  attr_reader :user, :params
+
+  def initialize(user, params)
+    @user   = user
+    @params = params
+  end
+
+  def call
+    Shixun.search(keyword,
+                  fields: search_fields,
+                  where: where_clauses,
+                  order: order_clauses,
+                  includes: includes_clauses,
+                  page: page,
+                  per_page: per_page)
+  end
+
+  private
+
+  def tag_filter_shixun_ids
+    return [] if params[:tag_level].to_i == 0 || params[:tag_id].blank?
+
+    case params[:tag_level].to_i
+    when 1 then
+      Repertoire.find(params[:tag_id]).tag_repertoires.joins(:shixun_tag_repertoires)
+        .pluck('shixun_tag_repertoires.shixun_id')
+    when 2 then
+      SubRepertoire.find(params[:tag_id]).tag_repertoires.joins(:shixun_tag_repertoires)
+        .pluck('shixun_tag_repertoires.shixun_id')
+    when 3 then
+      TagRepertoire.find(params[:tag_id]).shixun_tag_repertoires.pluck(:shixun_id)
+    else
+      []
+    end
+  end
+
+  def user_filter_shixun_ids
+    return [] if params[:order_by] != 'mine'
+
+    user.shixun_members.pluck(:shixun_id) + user.myshixuns.pluck(:shixun_id)
+  end
+
+  def keyword
+    params[:keyword].to_s.strip.presence || '*'
+  end
+
+  def search_fields
+    %w(name^10 author_name challenge_names description challenge_tag_names)
+  end
+
+  def where_clauses
+    hash = {}
+
+    ids = user_filter_shixun_ids + tag_filter_shixun_ids
+    hash[:id] = ids if ids.present?
+
+    if params[:order_by] == 'mine'
+      hash[:status] = { not: -1 }
+    else
+      hash.merge!(hidden: false, status: 2)
+    end
+
+    unless params[:status].to_i.zero?
+      params[:status] = [0, 1] if params[:status].to_i == 1
+      hash[:status] = params[:status]
+    end
+
+    hash[:trainee] = params[:diff].to_i unless params[:diff].to_i.zero?
+
+    hash
+  end
+
+  def includes_clauses
+    []
+  end
+
+  def order_clauses
+    hash = { _score: :desc }
+    publish_order = { type: 'number', order: :desc, script: 'doc["status"].value=="2" ? 1 : 0' }
+
+    sort = params[:sort].to_s.strip == 'asc' ? 'asc' : 'desc'
+    clauses =
+      case params[:order_by].presence
+      when 'new'  then { _script: publish_order, created_at: sort }
+      when 'hot'  then { _script: publish_order, myshixuns_count: sort }
+      when 'mine' then { created_at: sort }
+      else { _script: publish_order, publish_time: sort }
+      end
+    hash.merge!(clauses)
+
+    hash
+  end
+end

app/views/searchs/index.json.jbuilder

@@ -0,0 +1,10 @@
+json.count @results.total_count
+json.results do
+  json.array! @results.with_highlights(multiple: true) do |obj, highlights|
+    json.merge! obj.to_searchable_json
+    json.type obj.class.name.downcase
+
+    json.title highlights.delete(:name)&.join('...') || obj.searchable_title
+    json.description highlights.values[0,5].each { |arr| arr.is_a?(Array) ? arr.join('...') : arr }.join('<br/>')
+  end
+end

app/views/shixuns/_shixun.json.jbuilder

@@ -15,7 +15,7 @@ json.array! shixuns do |shixun|
   json.status shixun.status
   json.power (current_user.shixun_permission(shixun)) # 现在首页只显示已发布的实训
   # REDO: 局部缓存
-  json.tag_name shixun.tag_repertoires.first.try(:name)
+  json.tag_name @tag_name_map&.fetch(shixun.id) || shixun.tag_repertoires.first.try(:name)
   json.myshixuns_count shixun.myshixuns_count
   json.stu_num shixun.myshixuns_count
   json.score_info shixun.averge_star

config/routes.rb

@@ -9,6 +9,8 @@ Rails.application.routes.draw do
     get 'home/index'
     get 'home/search'
 
+    get 'search', to: 'searchs#index'
+
     post   'praise_tread/like',   to: 'praise_tread#like'
     delete 'praise_tread/unlike', to: 'praise_tread#unlike'