ES search feature

dev_forum
p31729568 5 years ago
parent f10cbf2250
commit 0a286bf5b0

@ -89,3 +89,5 @@ gem 'sinatra'
# batch insert
gem 'bulk_insert'
# elasticsearch
gem 'searchkick'

@ -90,6 +90,14 @@ GEM
connection_pool (2.2.2)
crass (1.0.4)
diff-lcs (1.3)
elasticsearch (7.1.0)
elasticsearch-api (= 7.1.0)
elasticsearch-transport (= 7.1.0)
elasticsearch-api (7.1.0)
multi_json
elasticsearch-transport (7.1.0)
faraday
multi_json
erubi (1.7.1)
execjs (2.7.0)
faraday (0.15.4)
@ -100,6 +108,7 @@ GEM
grape-entity (0.7.1)
activesupport (>= 4.0)
multi_json (>= 1.3.2)
hashie (3.6.0)
htmlentities (4.3.4)
httparty (0.16.2)
multi_xml (>= 0.5.2)
@ -255,6 +264,10 @@ GEM
sprockets (>= 2.8, < 4.0)
sprockets-rails (>= 2.0, < 4.0)
tilt (>= 1.1, < 3)
searchkick (3.1.3)
activemodel (>= 4.2)
elasticsearch (>= 5)
hashie
selenium-webdriver (3.14.0)
childprocess (~> 0.5)
rubyzip (~> 1.2)
@ -344,6 +357,7 @@ DEPENDENCIES
ruby-ole
rubyzip
sass-rails (~> 5.0)
searchkick
selenium-webdriver
sidekiq
simple_xlsx_reader

@ -410,10 +410,8 @@ class HomeworkCommonsController < ApplicationController
homework_detail_group = @homework.homework_detail_group
param_min = params[:min_num].to_i
param_max = params[:max_num].to_i
homework_detail_group.min_num = @homework.has_commit_work ? (param_min > homework_detail_group.min_num ? homework_detail_group.min_num :
param_min) : param_min
homework_detail_group.max_num = @homework.has_commit_work ? (param_max < homework_detail_group.max_num ? homework_detail_group.max_num :
param_max) : param_max
homework_detail_group.min_num = @homework.has_commit_work ? [param_min, homework_detail_group.min_num].min : param_min
homework_detail_group.max_num = @homework.has_commit_work ? [param_max, homework_detail_group.max_num].max : param_max
homework_detail_group.base_on_project = params[:base_on_project] unless @homework.has_relate_project
homework_detail_group.save!
end

@ -0,0 +1,10 @@
class SearchsController < ApplicationController
def index
@results = SearchService.call(search_params)
end
private
def search_params
params.permit(:keyword, :type, :page, :per_page)
end
end

@ -1,4 +1,7 @@
class ShixunsController < ApplicationController
include ShixunsHelper
include ApplicationHelper
before_action :require_login, :check_auth, except: [:download_file, :index, :menus]
before_action :check_auth, except: [:download_file, :index, :menus]
@ -14,9 +17,6 @@ class ShixunsController < ApplicationController
before_action :special_allowed, only: [:send_to_course, :search_user_courses]
include ShixunsHelper
include ApplicationHelper
## 获取课程列表
def index
## 我的实训
@ -59,7 +59,7 @@ class ShixunsController < ApplicationController
end
## 筛选 难度
if params[:diff].present? && params[:diff].to_i != 0
if params[:diff].present? && params[:diff].to_i != 0
@shixuns = @shixuns.where(trainee: params[:diff])
end
@ -84,6 +84,12 @@ class ShixunsController < ApplicationController
limit = params[:limit] || 16
@shixuns = @shixuns.includes(:tag_repertoires, :challenges).page(page).per(limit)
@tag_name_map = TagRepertoire.joins(:shixun_tag_repertoires)
.where(shixun_tag_repertoires: { shixun_id: @shixuns.map(&:id) })
.group('shixun_tag_repertoires.shixun_id')
.select('shixun_id, tag_repertoires.name')
.each_with_object({}) { |r, obj| obj[r.shixun_id] = r.name }
end
## 获取顶部菜单

@ -33,4 +33,12 @@ module Util
Rails.logger.error(exception.message)
exception.backtrace.each { |message| Rails.logger.error(message) }
end
def map_or_pluck(relation, name)
relation.is_a?(Array) || relation.loaded? ? relation.map(&name.to_sym) : relation.pluck(name)
end
def extract_content(str)
str.gsub(/<\/?.*?>/, '').gsub(/[\n\t\r]/, '').gsub(/&nbsp;/, '')
end
end

@ -116,5 +116,4 @@ class Challenge < ApplicationRecord
end
# 关卡评测文件
end

@ -1,4 +1,6 @@
class ChallengeTag < ApplicationRecord
# TODO: ES feature
# include Searchable::Dependents::ChallengeTag
belongs_to :challenge, counter_cache: true
belongs_to :challenge_choose, optional: true

@ -1,4 +1,7 @@
class Course < ApplicationRecord
# TODO: ES feature
# include Searchable::Course
has_many :boards, dependent: :destroy
belongs_to :teacher, class_name: 'User', foreign_key: :tea_id # 定义一个方法teacher该方法通过tea_id来调用User表
@ -22,6 +25,8 @@ class Course < ApplicationRecord
has_many :graduation_groups, dependent: :destroy
has_many :course_members, dependent: :destroy
has_many :teacher_course_members, -> { teachers_and_admin }, class_name: 'CourseMember'
has_many :teacher_users, through: :teacher_course_members, source: :user
has_many :course_messages, dependent: :destroy
has_many :homework_commons, dependent: :destroy
has_many :homework_group_settings

@ -1,4 +1,6 @@
class Memo < ApplicationRecord
# TODO: ES feature
# include Searchable::Memo
has_many :memo_tag_repertoires, :dependent => :destroy
has_many :tag_repertoires, :through => :memo_tag_repertoires
@ -9,6 +11,9 @@ class Memo < ApplicationRecord
belongs_to :author, class_name: 'User', foreign_key: 'author_id'
belongs_to :parent, class_name: 'Memo', foreign_key: 'parent_id'
has_many :descendants, foreign_key: :root_id, class_name: 'Memo'
has_many :children, foreign_key: :parent_id, class_name: 'Memo'
scope :field_for_list, lambda{
select([:id, :subject, :author_id, :sticky, :updated_at, :language, :reward, :all_replies_count, :viewed_count, :forum_id])
}

@ -0,0 +1,3 @@
module Searchable
MAXIMUM_LENGTH = 10922 # 最大字节数为32766 一个汉字3个字节
end

@ -0,0 +1,37 @@
module Searchable::Course
extend ActiveSupport::Concern
included do
searchkick language: 'chinese', callbacks: :async
scope :search_import, -> { includes(:teacher_users, teacher: { user_extension: :school } ) }
end
def searchable_title
name
end
def search_data
{
name: name,
author_name: teacher&.real_name
}
end
def to_searchable_json
{
id: id,
author_name: teacher.real_name,
author_school_name: teacher.school_name,
visits_count: visits,
members_count: members_count,
is_public: is_public == 1
}
end
module ClassMethods
def searchable_includes
{ teacher: { user_extension: :school } }
end
end
end

@ -0,0 +1,2 @@
module Searchable::Dependents
end

@ -0,0 +1,16 @@
module Searchable::Dependents::ChallengeTag
extend ActiveSupport::Concern
included do
after_create_commit :check_searchable_dependents
after_update_commit :check_searchable_dependents
end
private
def check_searchable_dependents
if new_record? || name_previously_changed?
challenge.shixun.reindex(:searchable_challenge_data)
end
end
end

@ -0,0 +1,15 @@
module Searchable::Dependents::Stage
extend ActiveSupport::Concern
included do
after_update_commit :check_searchable_dependents
end
private
def check_searchable_dependents
if name_previously_changed? || description_previously_changed?
subject.reindex(:searchable_stages_data)
end
end
end

@ -0,0 +1,22 @@
module Searchable::Dependents::User
extend ActiveSupport::Concern
included do
after_update_commit :check_searchable_dependents
end
private
def check_searchable_dependents
if firstname_previously_changed? || lastname_previously_changed? || user_extension.school_id_previously_changed?
# reindex shixun
created_shixuns.each{ |shixun| shixun.reindex(:searchable_user_data) }
# reindex course
manage_courses.each(&:reindex)
# reindex subject
created_subjects.each { |subject| subject.reindex(:searchable_user_data) }
end
end
end

@ -0,0 +1,46 @@
module Searchable::Memo
extend ActiveSupport::Concern
included do
searchkick language: 'chinese', callbacks: :async
scope :search_import, -> { includes(:descendants) }
end
def searchable_title
subject
end
def should_index?
hidden.zero? && root_id.blank? && parent_id.blank?
end
def search_data
{
name: subject,
content: Util.extract_content(content)[0..Searchable::MAXIMUM_LENGTH],
}.merge!(searchable_descendants_data)
end
def searchable_descendants_data
{
descendants_contents: Util.map_or_pluck(descendants, :content)
.map { |content| Util.extract_content(content)[0..Searchable::MAXIMUM_LENGTH] }
}
end
def to_searchable_json
{
id: id,
author_name: author.full_name,
visits_count: viewed_count,
all_replies_count: all_replies_count
}
end
module ClassMethods
def searchable_includes
[:author]
end
end
end

@ -0,0 +1,58 @@
module Searchable::Shixun
extend ActiveSupport::Concern
included do
searchkick language: 'chinese'#, callbacks: :async
scope :search_import, -> { includes(:shixun_info, :challenges, :challenge_tags, :users, user: { user_extension: :school }) }
end
def searchable_title
name
end
def search_data
{
name: name,
description: Util.extract_content(description)[0..Searchable::MAXIMUM_LENGTH]
}.merge!(searchable_user_data)
.merge!(searchable_challenge_data)
end
def searchable_user_data
{
author_name: user.real_name,
author_school_name: user.school_name,
}
end
def searchable_challenge_data
challenge_names = Util.map_or_pluck(challenges, :subject)
.each_with_index.map { |subject, index| "#{index + 1}#{subject}" }
{
challenge_names: challenge_names,
challenge_tag_names: Util.map_or_pluck(challenge_tags, :name).uniq.join(' ')
}
end
def should_index?
status == 2 # published
end
def to_searchable_json
{
id: id,
author_name: user.real_name,
author_school_name: user.school_name,
visits_count: visits,
challenges_count: challenges_count
}
end
module ClassMethods
def searchable_includes
{ user: { user_extension: :school } }
end
end
end

@ -0,0 +1,61 @@
module Searchable::Subject
extend ActiveSupport::Concern
included do
searchkick language: 'chinese', callbacks: :async
scope :search_import, -> { includes(:users, :stages, user: { user_extension: :school }) }
end
def searchable_title
name
end
def should_index?
!hidden? && status == 2 # published
end
def search_data
{
name: name,
description: Util.extract_content(description)[0..Searchable::MAXIMUM_LENGTH]
}.merge!(searchable_user_data)
.merge!(searchable_stages_data)
end
def searchable_user_data
{
author_name: user.real_name,
author_school_name: user.school_name,
}
end
def searchable_stages_data
subject_stages =
stages.map do |stage|
{
name: stage.name,
description: Util.extract_content(stage.description)[0..Searchable::MAXIMUM_LENGTH]
}
end
{ subject_stages: subject_stages}
end
def to_searchable_json
{
id: id,
author_name: user.real_name,
author_school_name: user.school_name,
visits_count: visits,
stage_count: stages_count,
stage_shixuns_count: stage_shixuns_count
}
end
module ClassMethods
def searchable_includes
{ user: { user_extension: :school } }
end
end
end

@ -1,8 +1,12 @@
class Shixun < ApplicationRecord
# TODO: ES feature
# include Searchable::Shixun
# status: 0编辑 1申请发布 2正式发布 3关闭 -1软删除
# hide_code 隐藏代码窗口
# code_hidden: 隐藏代码目录
has_many :challenges, dependent: :destroy
has_many :challenge_tags, through: :challenges
has_many :myshixuns, :dependent => :destroy
has_many :shixun_members, dependent: :destroy
has_many :users, through: :shixun_members
@ -35,7 +39,6 @@ class Shixun < ApplicationRecord
# 实训服务配置
has_many :shixun_service_configs, :dependent => :destroy
scope :search_by_name, ->(keyword) { where("name like ? or description like ? ",
"%#{keyword}%", "%#{keyword}%") }

@ -1,4 +1,7 @@
class Stage < ApplicationRecord
# TODO: ES feature
# include Searchable::Dependents::Stage
belongs_to :subject, counter_cache: true
has_many :stage_shixuns, -> { order("stage_shixuns.position ASC") }, dependent: :destroy

@ -2,6 +2,9 @@
# 可以在初始创建的时候
class Subject < ApplicationRecord
# TODO: ES feature
# include Searchable::Subject
#status :0 编辑中 1 审核中 2 发布
belongs_to :repertoire
belongs_to :user

@ -1,5 +1,8 @@
class User < ApplicationRecord
include Watchable
# TODO: ES feature
# include Searchable::Dependents::User
# Account statuses
STATUS_ANONYMOUS = 0
STATUS_ACTIVE = 1
@ -28,6 +31,7 @@ class User < ApplicationRecord
accepts_nested_attributes_for :user_extension, update_only: true
has_many :memos, foreign_key: 'author_id'
has_many :created_shixuns, class_name: 'Shixun'
has_many :shixun_members, :dependent => :destroy
has_many :shixuns, :through => :shixun_members
has_many :myshixuns, :dependent => :destroy
@ -55,6 +59,7 @@ class User < ApplicationRecord
has_many :tidings, :dependent => :destroy
has_many :games, :dependent => :destroy
has_many :created_subjects
has_many :subjects, :through => :subject_members
has_many :subject_members, :dependent => :destroy
has_many :grades, :dependent => :destroy

@ -2,7 +2,7 @@ class UserExtension < ApplicationRecord
# identity 0: 教师教授 1: 学生, 2: 专业人士, 3: 开发者
enum identity: { teacher: 0, student: 1, professional: 2, developer: 3 }
belongs_to :user
belongs_to :user, touch: true
belongs_to :school
belongs_to :department, optional: true

@ -0,0 +1,41 @@
module ElasticsearchAble
extend ActiveSupport::Concern
private
def default_options
{
debug: Rails.env.development?,
highlight: highlight_options,
body_options: body_options,
page: page,
per_page: per_page
}
end
def keyword
params[:keyword].to_s.strip.presence || '*'
end
def highlight_options
{
fragment_size: EduSetting.get('es_highlight_fragment_size') || 30,
tag: '<span>'
}
end
def body_options
{
min_score: EduSetting.get('es_min_score') || 10
}
end
def per_page
per_page = params[:per_page].to_s.strip.presence || params[:limit].to_s.strip.presence
per_page.to_i <= 0 ? 20 : per_page.to_i
end
def page
params[:page].to_i <= 0 ? 1 : params[:page].to_i
end
end

@ -0,0 +1,39 @@
class SearchService < ApplicationService
include ElasticsearchAble
attr_reader :params
def initialize(params)
@params = params
end
def call
Searchkick.search(keyword, search_options)
end
private
def search_options
{
index_name: index_names,
model_includes: model_includes
}.merge(default_options)
end
def index_names
@_index_names ||=
case params[:type].to_s.strip
when 'shixun' then [Shixun]
when 'course' then [Course]
when 'subject' then [Subject]
when 'memo' then [Memo]
else [Shixun, Course, Subject, Memo]
end
end
def model_includes
index_names.each_with_object({}) do |klass, obj|
obj[klass] = klass.searchable_includes
end
end
end

@ -0,0 +1,96 @@
class SearchShixunService < ApplicationService
include ElasticsearchAble
attr_reader :user, :params
def initialize(user, params)
@user = user
@params = params
end
def call
Shixun.search(keyword,
fields: search_fields,
where: where_clauses,
order: order_clauses,
includes: includes_clauses,
page: page,
per_page: per_page)
end
private
def tag_filter_shixun_ids
return [] if params[:tag_level].to_i == 0 || params[:tag_id].blank?
case params[:tag_level].to_i
when 1 then
Repertoire.find(params[:tag_id]).tag_repertoires.joins(:shixun_tag_repertoires)
.pluck('shixun_tag_repertoires.shixun_id')
when 2 then
SubRepertoire.find(params[:tag_id]).tag_repertoires.joins(:shixun_tag_repertoires)
.pluck('shixun_tag_repertoires.shixun_id')
when 3 then
TagRepertoire.find(params[:tag_id]).shixun_tag_repertoires.pluck(:shixun_id)
else
[]
end
end
def user_filter_shixun_ids
return [] if params[:order_by] != 'mine'
user.shixun_members.pluck(:shixun_id) + user.myshixuns.pluck(:shixun_id)
end
def keyword
params[:keyword].to_s.strip.presence || '*'
end
def search_fields
%w(name^10 author_name challenge_names description challenge_tag_names)
end
def where_clauses
hash = {}
ids = user_filter_shixun_ids + tag_filter_shixun_ids
hash[:id] = ids if ids.present?
if params[:order_by] == 'mine'
hash[:status] = { not: -1 }
else
hash.merge!(hidden: false, status: 2)
end
unless params[:status].to_i.zero?
params[:status] = [0, 1] if params[:status].to_i == 1
hash[:status] = params[:status]
end
hash[:trainee] = params[:diff].to_i unless params[:diff].to_i.zero?
hash
end
def includes_clauses
[]
end
def order_clauses
hash = { _score: :desc }
publish_order = { type: 'number', order: :desc, script: 'doc["status"].value=="2" ? 1 : 0' }
sort = params[:sort].to_s.strip == 'asc' ? 'asc' : 'desc'
clauses =
case params[:order_by].presence
when 'new' then { _script: publish_order, created_at: sort }
when 'hot' then { _script: publish_order, myshixuns_count: sort }
when 'mine' then { created_at: sort }
else { _script: publish_order, publish_time: sort }
end
hash.merge!(clauses)
hash
end
end

@ -0,0 +1,10 @@
json.count @results.total_count
json.results do
json.array! @results.with_highlights(multiple: true) do |obj, highlights|
json.merge! obj.to_searchable_json
json.type obj.class.name.downcase
json.title highlights.delete(:name)&.join('...') || obj.searchable_title
json.description highlights.values[0,5].each { |arr| arr.is_a?(Array) ? arr.join('...') : arr }.join('<br/>')
end
end

@ -15,7 +15,7 @@ json.array! shixuns do |shixun|
json.status shixun.status
json.power (current_user.shixun_permission(shixun)) # 现在首页只显示已发布的实训
# REDO: 局部缓存
json.tag_name shixun.tag_repertoires.first.try(:name)
json.tag_name @tag_name_map&.fetch(shixun.id) || shixun.tag_repertoires.first.try(:name)
json.myshixuns_count shixun.myshixuns_count
json.stu_num shixun.myshixuns_count
json.score_info shixun.averge_star

@ -9,6 +9,8 @@ Rails.application.routes.draw do
get 'home/index'
get 'home/search'
get 'search', to: 'searchs#index'
post 'praise_tread/like', to: 'praise_tread#like'
delete 'praise_tread/unlike', to: 'praise_tread#unlike'

Loading…
Cancel
Save