# bjy: 导入时间模块 import time # bjy: 导入Elasticsearch的客户端模块和异常类 import elasticsearch.client import elasticsearch.exceptions # bjy: 导入Django的设置 from django.conf import settings # bjy: 从elasticsearch_dsl中导入文档、内部文档、字段类型和连接管理器 from elasticsearch_dsl import Document, InnerDoc, Date, Integer, Long, Text, Object, GeoPoint, Keyword, Boolean from elasticsearch_dsl.connections import connections # bjy: 从blog应用中导入Article模型 from blog.models import Article # bjy: 检查Django设置中是否配置了ELASTICSEARCH_DSL，以决定是否启用Elasticsearch功能 ELASTICSEARCH_ENABLED = hasattr(settings, 'ELASTICSEARCH_DSL') # bjy: 如果启用了Elasticsearch if ELASTICSEARCH_ENABLED: # bjy: 根据Django设置创建到Elasticsearch的连接 connections.create_connection( hosts=[settings.ELASTICSEARCH_DSL['default']['hosts']]) # bjy: 导入并实例化Elasticsearch客户端 from elasticsearch import Elasticsearch es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) # bjy: 导入并实例化Ingest客户端，用于管理管道 from elasticsearch.client import IngestClient c = IngestClient(es) # bjy: 尝试获取名为'geoip'的管道 try: c.get_pipeline('geoip') # bjy: 如果管道不存在，则创建它 except elasticsearch.exceptions.NotFoundError: # bjy: 创建一个geoip管道，用于根据IP地址添加地理位置信息 c.put_pipeline('geoip', body='''{ "description" : "Add geoip info", "processors" : [ { "geoip" : { "field" : "ip" } } ] }''') # bjy: 定义一个内部文档（InnerDoc）结构，用于存储IP地理位置信息 class GeoIp(InnerDoc): # bjy: 大洲名称 continent_name = Keyword() # bjy: 国家ISO代码 country_iso_code = Keyword() # bjy: 国家名称 country_name = Keyword() # bjy: 地理坐标（经纬度） location = GeoPoint() # bjy: 定义内部文档，用于存储用户代理（User-Agent）中的浏览器信息 class UserAgentBrowser(InnerDoc): # bjy: 浏览器家族（如Chrome, Firefox） Family = Keyword() # bjy: 浏览器版本 Version = Keyword() # bjy: 定义内部文档，用于存储用户代理中的操作系统信息 class UserAgentOS(UserAgentBrowser): # bjy: 继承自UserAgentBrowser，结构相同 pass # bjy: 定义内部文档，用于存储用户代理中的设备信息 class UserAgentDevice(InnerDoc): # bjy: 设备家族（如iPhone, Android） Family = Keyword() # bjy: 设备品牌（如Apple, Samsung） Brand = Keyword() # bjy: 设备型号（如iPhone 12） Model = Keyword() # bjy: 定义内部文档，用于存储完整的用户代理信息 class UserAgent(InnerDoc): # bjy: 嵌套浏览器信息 browser = Object(UserAgentBrowser, required=False) # bjy: 嵌套操作系统信息 os = Object(UserAgentOS, required=False) # bjy: 嵌套设备信息 device = Object(UserAgentDevice, required=False) # bjy: 原始User-Agent字符串 string = Text() # bjy: 是否为爬虫或机器人 is_bot = Boolean() # bjy: 定义一个Elasticsearch文档，用于存储页面性能数据（如响应时间） class ElapsedTimeDocument(Document): # bjy: 请求的URL url = Keyword() # bjy: 请求耗时（毫秒） time_taken = Long() # bjy: 日志记录时间 log_datetime = Date() # bjy: 客户端IP地址 ip = Keyword() # bjy: 嵌套的IP地理位置信息 geoip = Object(GeoIp, required=False) # bjy: 嵌套的用户代理信息 useragent = Object(UserAgent, required=False) class Index: # bjy: 指定索引名称为'performance' name = 'performance' # bjy: 设置索引的分片和副本数 settings = { "number_of_shards": 1, "number_of_replicas": 0 } class Meta: # bjy: 指定文档类型 doc_type = 'ElapsedTime' # bjy: 定义一个管理类，用于操作ElapsedTimeDocument索引 class ElaspedTimeDocumentManager: @staticmethod def build_index(): # bjy: 如果索引不存在，则创建它 from elasticsearch import Elasticsearch client = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) res = client.indices.exists(index="performance") if not res: ElapsedTimeDocument.init() @staticmethod def delete_index(): # bjy: 删除'performance'索引 from elasticsearch import Elasticsearch es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) es.indices.delete(index='performance', ignore=[400, 404]) @staticmethod def create(url, time_taken, log_datetime, useragent, ip): # bjy: 确保索引存在 ElaspedTimeDocumentManager.build_index() # bjy: 构建UserAgent内部文档对象 ua = UserAgent() ua.browser = UserAgentBrowser() ua.browser.Family = useragent.browser.family ua.browser.Version = useragent.browser.version_string ua.os = UserAgentOS() ua.os.Family = useragent.os.family ua.os.Version = useragent.os.version_string ua.device = UserAgentDevice() ua.device.Family = useragent.device.family ua.device.Brand = useragent.device.brand ua.device.Model = useragent.device.model ua.string = useragent.ua_string ua.is_bot = useragent.is_bot # bjy: 创建ElapsedTimeDocument文档实例 doc = ElapsedTimeDocument( meta={ # bjy: 使用当前时间的毫秒数作为文档ID 'id': int( round( time.time() * 1000)) }, url=url, time_taken=time_taken, log_datetime=log_datetime, useragent=ua, ip=ip) # bjy: 保存文档，并使用'geoip'管道处理IP地址 doc.save(pipeline="geoip") # bjy: 定义一个Elasticsearch文档，用于存储博客文章数据，以支持全文搜索 class ArticleDocument(Document): # bjy: 文章内容，使用ik分词器进行索引和搜索 body = Text(analyzer='ik_max_word', search_analyzer='ik_smart') # bjy: 文章标题，使用ik分词器 title = Text(analyzer='ik_max_word', search_analyzer='ik_smart') # bjy: 作者信息，为一个对象类型 author = Object(properties={ 'nickname': Text(analyzer='ik_max_word', search_analyzer='ik_smart'), 'id': Integer() }) # bjy: 分类信息，为一个对象类型 category = Object(properties={ 'name': Text(analyzer='ik_max_word', search_analyzer='ik_smart'), 'id': Integer() }) # bjy: 标签信息，为一个对象类型 tags = Object(properties={ 'name': Text(analyzer='ik_max_word', search_analyzer='ik_smart'), 'id': Integer() }) # bjy: 发布时间 pub_time = Date() # bjy: 文章状态 status = Text() # bjy: 评论状态 comment_status = Text() # bjy: 文章类型 type = Text() # bjy: 浏览量 views = Integer() # bjy: 文章排序权重 article_order = Integer() class Index: # bjy: 指定索引名称为'blog' name = 'blog' # bjy: 设置索引的分片和副本数 settings = { "number_of_shards": 1, "number_of_replicas": 0 } class Meta: # bjy: 指定文档类型 doc_type = 'Article' # bjy: 定义一个管理类，用于操作ArticleDocument索引 class ArticleDocumentManager(): def __init__(self): # bjy: 初始化时创建索引 self.create_index() def create_index(self): # bjy: 创建'blog'索引 ArticleDocument.init() def delete_index(self): # bjy: 删除'blog'索引 from elasticsearch import Elasticsearch es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) es.indices.delete(index='blog', ignore=[400, 404]) def convert_to_doc(self, articles): # bjy: 将Django的Article查询集转换为ArticleDocument对象列表 return [ ArticleDocument( meta={ 'id': article.id}, body=article.body, title=article.title, author={ 'nickname': article.author.username, 'id': article.author.id}, category={ 'name': article.category.name, 'id': article.category.id}, tags=[ { 'name': t.name, 'id': t.id} for t in article.tags.all()], pub_time=article.pub_time, status=article.status, comment_status=article.comment_status, type=article.type, views=article.views, article_order=article.article_order) for article in articles] def rebuild(self, articles=None): # bjy: 重建索引。如果未提供articles，则使用所有文章 ArticleDocument.init() articles = articles if articles else Article.objects.all() docs = self.convert_to_doc(articles) # bjy: 遍历并保存每个文档 for doc in docs: doc.save() def update_docs(self, docs): # bjy: 更新一组文档 for doc in docs: doc.save()