Merge pull request #462 from liangliangyy/dev

增加es测试
5 years ago · 1008ee62b7
parent 8390180ace 75f1081f37
commit 1008ee62b7
8 changed files with 86 additions and 66 deletions
--- a/.coveragerc
+++ b/.coveragerc
@ -7,7 +7,6 @@ omit =
    *.html
    *whoosh_cn_backend*
    *apps*
-    *commands*
    *oauth*
    *travis_test*
    *settings.py*
--- a/.github/workflows/django.yml
+++ b/.github/workflows/django.yml
@ -13,44 +13,57 @@ jobs:
    strategy:
      max-parallel: 4
      matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [ 3.6, 3.7, 3.8, 3.9 ]

    steps:
-    - name: Start MySQL
-      uses: samin/mysql-action@v1.3
-      with:
-        # The port of host
-        host port: 3306 # optional, default is 3306
-        # The port of container
-        container port: 3306 # optional, default is 3306
-        # --character-set-server - The character set of MySQL server
-        character set server: utf8mb4 # optional, default is utf8mb4
-        # --collation-server - The character collation of MySQL server
-        collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci
-        # Version of MySQL to use
-        mysql version: latest # optional, default is latest
-        # MYSQL_ROOT_PASSWORD - root superuser password
-        mysql root password: root # optional, default is 
-        # MYSQL_DATABASE - name for the default database that is created
-        mysql database: djangoblog # optional, default is 
-        # MYSQL_USER - create the specified user with superuser power for created database
-        mysql user: root # optional, default is 
-        # MYSQL_PASSWORD - specified superuser password which user is power for created database
-        mysql password: root # optional, default is 
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Run Tests
-      env:
-        DJANGO_MYSQL_PASSWORD:  root
-        DJANGO_MYSQL_HOST: 127.0.0.1
-      run: |
-        python manage.py makemigrations
-        python manage.py migrate
-        python manage.py test
+      - name: Start MySQL
+        uses: samin/mysql-action@v1.3
+        with:
+          # The port of host
+          host port: 3306 # optional, default is 3306
+          # The port of container
+          container port: 3306 # optional, default is 3306
+          # --character-set-server - The character set of MySQL server
+          character set server: utf8mb4 # optional, default is utf8mb4
+          # --collation-server - The character collation of MySQL server
+          collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci
+          # Version of MySQL to use
+          mysql version: latest # optional, default is latest
+          # MYSQL_ROOT_PASSWORD - root superuser password
+          mysql root password: root # optional, default is
+          # MYSQL_DATABASE - name for the default database that is created
+          mysql database: djangoblog # optional, default is
+          # MYSQL_USER - create the specified user with superuser power for created database
+          mysql user: root # optional, default is
+          # MYSQL_PASSWORD - specified superuser password which user is power for created database
+          mysql password: root # optional, default is
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+
+      - uses: miyataka/elasticsearch-github-actions@1
+        with:
+          stack-version: '7.12.1'
+          plugins: 'https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.12.1/elasticsearch-analysis-ik-7.12.1.zip'
+
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run Tests
+        env:
+          DJANGO_MYSQL_PASSWORD: root
+          DJANGO_MYSQL_HOST: 127.0.0.1
+          DJANGO_ELASTICSEARCH_HOST: 127.0.0.1:9200
+        run: |
+          python manage.py makemigrations
+          python manage.py migrate
+          python manage.py test
--- a/DjangoBlog/settings.py
+++ b/DjangoBlog/settings.py
@ -195,7 +195,7 @@ CACHES = {

 SITE_ID = 1
 BAIDU_NOTIFY_URL = os.environ.get('DJANGO_BAIDU_NOTIFY_URL') \
-    or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn'
+                   or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn'

 # Email:
 EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
@ -293,3 +293,12 @@ COMPRESS_JS_FILTERS = [
 MEDIA_ROOT = os.path.join(BASE_DIR, 'uploads')
 MEDIA_URL = '/media/'
 X_FRAME_OPTIONS = 'SAMEORIGIN'
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+if os.environ.get('DJANGO_ELASTICSEARCH_HOST'):
+    ELASTICSEARCH_DSL = {
+        'default': {
+            'hosts': os.environ.get('DJANGO_ELASTICSEARCH_HOST')
+        },
+    }
--- a/blog/documents.py
+++ b/blog/documents.py
@ -28,8 +28,7 @@ class ElapsedTimeDocument(Document):
    url = Text()
    time_taken = Long()
    log_datetime = Date()
-    type = Text(analyzer='ik_max_word')
-    useragent = Text()
+    useragent = Text(analyzer='ik_max_word', search_analyzer='ik_smart')
    ip = Text()

    class Index:
@ -44,9 +43,14 @@ class ElapsedTimeDocument(Document):


 class ElaspedTimeDocumentManager():
+    @staticmethod
+    def delete_index():
+        from elasticsearch import Elasticsearch
+        es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts'])
+        es.indices.delete(index='performance', ignore=[400, 404])

    @staticmethod
-    def create(url, time_taken, log_datetime, type, useragent, ip):
+    def create(url, time_taken, log_datetime, useragent, ip):
        # if not hasattr(ElaspedTimeDocumentManager, 'mapping_created'):
        #     ElapsedTimeDocument.init()
        #     setattr(ElaspedTimeDocumentManager, 'mapping_created', True)
@ -59,7 +63,6 @@ class ElaspedTimeDocumentManager():
            url=url,
            time_taken=time_taken,
            log_datetime=log_datetime,
-            type=type,
            useragent=useragent, ip=ip)
        doc.save()

--- a/blog/management/commands/build_index.py
+++ b/blog/management/commands/build_index.py
@ -11,7 +11,7 @@
@time: 2019-04-20 20:39
 """

-from blog.documents import ElapsedTimeDocument, ArticleDocumentManager
+from blog.documents import ElapsedTimeDocument, ArticleDocumentManager, ElaspedTimeDocumentManager

 from django.core.management.base import BaseCommand
 from blog.models import Article
@ -22,9 +22,9 @@ class Command(BaseCommand):
    help = 'build search index'

    def handle(self, *args, **options):
+        manager = ElapsedTimeDocument()
+        ElaspedTimeDocumentManager.delete_index()
+        manager.init()
        manager = ArticleDocumentManager()
        manager.delete_index()
        manager.rebuild()
-
-        manager = ElapsedTimeDocument()
-        manager.init()
--- a/blog/middleware.py
+++ b/blog/middleware.py
@ -44,7 +44,6 @@ class OnlineMiddleware(object):
                        url=url,
                        time_taken=time_taken,
                        log_datetime=timezone.now(),
-                        type='blog',
                        useragent=http_user_agent,
                        ip=ip)
                response.content = response.content.replace(
--- a/blog/tests.py
+++ b/blog/tests.py
@ -199,12 +199,11 @@ class ArticleTest(TestCase):
        self.assertEqual(rsp.status_code, 404)

    def test_commands(self):
+        from blog.documents import ELASTICSEARCH_ENABLED
+        if ELASTICSEARCH_ENABLED:
+            call_command("build_index")
        call_command("ping_baidu", "all")
        call_command("create_testdata")
        call_command("clear_cache")
        call_command("sync_user_avatar")
        call_command("build_search_words")
-
-        from blog.documents import ELASTICSEARCH_ENABLED
-        if ELASTICSEARCH_ENABLED:
-            call_command("build_index")
--- a/requirements.txt
+++ b/requirements.txt
@ -1,42 +1,40 @@
-coverage==5.4
-Django==3.1.8
+coverage==5.5
+Django==3.2.2
 django-appconf==1.0.4
 django-autoslug==1.9.8
-django-compressor==2.4
+django-compressor==2.4.1
 django-debug-toolbar==3.2.1
 django-haystack==3.0
 django-ipware==3.0.2
 django-mdeditor==0.1.18
 django-uuslug==1.2.0
-elasticsearch==7.11.0
+elasticsearch==7.12.1
 elasticsearch-dsl==7.3.0
 gevent==21.1.2
 greenlet==0.4.17
-importlib-metadata==3.7.0
 ipaddress==1.0.23
-isort==5.7.0
+isort==5.8.0
 jieba==0.42.1
 jsonpickle==2.0.0
-lazy-object-proxy==1.5.2
+lazy-object-proxy==1.6.0
 markdown2==2.4.0
 mccabe==0.6.1
 mistune==0.8.4
 mysqlclient==2.0.3
-Pillow==8.1.1
+Pillow==8.2.0
 pycparser==2.20
-Pygments==2.8.0
-pylint==2.7.1
+Pygments==2.9.0
+pylint==2.8.2
 pyparsing==2.4.7
 python-dateutil==2.8.1
 python-logstash==0.4.6
 python-memcached==1.59
-python-slugify==4.0.1
+python-slugify==5.0.2
 pytz==2021.1
 raven==6.10.0
 rcssmin==1.0.6
 requests==2.25.1
 rjsmin==1.1.0
-six==1.15.0
 urllib3==1.26.4
 WeRoBot==1.13.1
 Whoosh==2.7.4