From 5964c32b78188e84566eedf4f27bef746613a79a Mon Sep 17 00:00:00 2001 From: liangliang Date: Fri, 14 May 2021 19:26:08 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/django.yml | 94 +++++++++++++++++++++--------------- DjangoBlog/settings.py | 11 ++++- blog/documents.py | 2 +- blog/tests.py | 7 ++- requirements.txt | 22 ++++----- 5 files changed, 79 insertions(+), 57 deletions(-) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 6f66747..f46afe0 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -13,44 +13,60 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [ 3.6, 3.7, 3.8, 3.9 ] steps: - - name: Start MySQL - uses: samin/mysql-action@v1.3 - with: - # The port of host - host port: 3306 # optional, default is 3306 - # The port of container - container port: 3306 # optional, default is 3306 - # --character-set-server - The character set of MySQL server - character set server: utf8mb4 # optional, default is utf8mb4 - # --collation-server - The character collation of MySQL server - collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci - # Version of MySQL to use - mysql version: latest # optional, default is latest - # MYSQL_ROOT_PASSWORD - root superuser password - mysql root password: root # optional, default is - # MYSQL_DATABASE - name for the default database that is created - mysql database: djangoblog # optional, default is - # MYSQL_USER - create the specified user with superuser power for created database - mysql user: root # optional, default is - # MYSQL_PASSWORD - specified superuser password which user is power for created database - mysql password: root # optional, default is - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Run Tests - env: - DJANGO_MYSQL_PASSWORD: root - DJANGO_MYSQL_HOST: 127.0.0.1 - run: | - python manage.py makemigrations - python manage.py migrate - python manage.py test + - name: Start MySQL + uses: samin/mysql-action@v1.3 + with: + # The port of host + host port: 3306 # optional, default is 3306 + # The port of container + container port: 3306 # optional, default is 3306 + # --character-set-server - The character set of MySQL server + character set server: utf8mb4 # optional, default is utf8mb4 + # --collation-server - The character collation of MySQL server + collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci + # Version of MySQL to use + mysql version: latest # optional, default is latest + # MYSQL_ROOT_PASSWORD - root superuser password + mysql root password: root # optional, default is + # MYSQL_DATABASE - name for the default database that is created + mysql database: djangoblog # optional, default is + # MYSQL_USER - create the specified user with superuser power for created database + mysql user: root # optional, default is + # MYSQL_PASSWORD - specified superuser password which user is power for created database + mysql password: root # optional, default is + - name: Configure sysctl limits + run: | + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + + - uses: getong/elasticsearch-action@v1.2 + with: + elasticsearch version: '7.12.1' + host port: 9200 + container port: 9200 + host node port: 9300 + node port: 9300 + discovery type: 'single-node' + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run Tests + env: + DJANGO_MYSQL_PASSWORD: root + DJANGO_MYSQL_HOST: 127.0.0.1 + DJANGO_ELASTICSEARCH_HOST: 127.0.0.1:9200 + run: | + python manage.py makemigrations + python manage.py migrate + python manage.py test diff --git a/DjangoBlog/settings.py b/DjangoBlog/settings.py index 750c2e9..8623787 100644 --- a/DjangoBlog/settings.py +++ b/DjangoBlog/settings.py @@ -195,7 +195,7 @@ CACHES = { SITE_ID = 1 BAIDU_NOTIFY_URL = os.environ.get('DJANGO_BAIDU_NOTIFY_URL') \ - or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn' + or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn' # Email: EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' @@ -293,3 +293,12 @@ COMPRESS_JS_FILTERS = [ MEDIA_ROOT = os.path.join(BASE_DIR, 'uploads') MEDIA_URL = '/media/' X_FRAME_OPTIONS = 'SAMEORIGIN' + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +if os.environ.get('DJANGO_ELASTICSEARCH_HOST'): + ELASTICSEARCH_DSL = { + 'default': { + 'hosts': os.environ.get('DJANGO_ELASTICSEARCH_HOST') + }, + } diff --git a/blog/documents.py b/blog/documents.py index 9fb15b6..e2b9da5 100644 --- a/blog/documents.py +++ b/blog/documents.py @@ -28,7 +28,7 @@ class ElapsedTimeDocument(Document): url = Text() time_taken = Long() log_datetime = Date() - type = Text(analyzer='ik_max_word') + type = Text(analyzer='ik_max_word', search_analyzer='ik_smart') useragent = Text() ip = Text() diff --git a/blog/tests.py b/blog/tests.py index ebc92f3..74cc9e3 100644 --- a/blog/tests.py +++ b/blog/tests.py @@ -199,12 +199,11 @@ class ArticleTest(TestCase): self.assertEqual(rsp.status_code, 404) def test_commands(self): + from blog.documents import ELASTICSEARCH_ENABLED + if ELASTICSEARCH_ENABLED: + call_command("build_index") call_command("ping_baidu", "all") call_command("create_testdata") call_command("clear_cache") call_command("sync_user_avatar") call_command("build_search_words") - - from blog.documents import ELASTICSEARCH_ENABLED - if ELASTICSEARCH_ENABLED: - call_command("build_index") diff --git a/requirements.txt b/requirements.txt index b3b51ae..55d2937 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,42 +1,40 @@ -coverage==5.4 -Django==3.1.8 +coverage==5.5 +Django==3.2.2 django-appconf==1.0.4 django-autoslug==1.9.8 -django-compressor==2.4 +django-compressor==2.4.1 django-debug-toolbar==3.2.1 django-haystack==3.0 django-ipware==3.0.2 django-mdeditor==0.1.18 django-uuslug==1.2.0 -elasticsearch==7.11.0 +elasticsearch==7.12.1 elasticsearch-dsl==7.3.0 gevent==21.1.2 greenlet==0.4.17 -importlib-metadata==3.7.0 ipaddress==1.0.23 -isort==5.7.0 +isort==5.8.0 jieba==0.42.1 jsonpickle==2.0.0 -lazy-object-proxy==1.5.2 +lazy-object-proxy==1.6.0 markdown2==2.4.0 mccabe==0.6.1 mistune==0.8.4 mysqlclient==2.0.3 -Pillow==8.1.1 +Pillow==8.2.0 pycparser==2.20 -Pygments==2.8.0 -pylint==2.7.1 +Pygments==2.9.0 +pylint==2.8.2 pyparsing==2.4.7 python-dateutil==2.8.1 python-logstash==0.4.6 python-memcached==1.59 -python-slugify==4.0.1 +python-slugify==5.0.2 pytz==2021.1 raven==6.10.0 rcssmin==1.0.6 requests==2.25.1 rjsmin==1.1.0 -six==1.15.0 urllib3==1.26.4 WeRoBot==1.13.1 Whoosh==2.7.4 \ No newline at end of file From 6d1ecbc36db316c2ab7deddb8715e2af1ae38b61 Mon Sep 17 00:00:00 2001 From: liangliang Date: Fri, 14 May 2021 19:36:04 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/django.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index f46afe0..afa3c0e 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -44,14 +44,11 @@ jobs: sudo sysctl -w fs.file-max=262144 sudo sysctl -w vm.max_map_count=262144 - - uses: getong/elasticsearch-action@v1.2 + - uses: miyataka/elasticsearch-github-actions@1 with: - elasticsearch version: '7.12.1' - host port: 9200 - container port: 9200 - host node port: 9300 - node port: 9300 - discovery type: 'single-node' + stack-version: '7.12.1' + plugins: 'https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.12.1/elasticsearch-analysis-ik-7.12.1.zip' + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 From 914a008360a6d49163c12ff34967166e75de9707 Mon Sep 17 00:00:00 2001 From: liangliang Date: Fri, 14 May 2021 19:56:58 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- blog/documents.py | 6 ++---- blog/management/commands/build_index.py | 5 ++--- blog/middleware.py | 1 - 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/blog/documents.py b/blog/documents.py index e2b9da5..745b3b7 100644 --- a/blog/documents.py +++ b/blog/documents.py @@ -28,8 +28,7 @@ class ElapsedTimeDocument(Document): url = Text() time_taken = Long() log_datetime = Date() - type = Text(analyzer='ik_max_word', search_analyzer='ik_smart') - useragent = Text() + useragent = Text(analyzer='ik_max_word', search_analyzer='ik_smart') ip = Text() class Index: @@ -46,7 +45,7 @@ class ElapsedTimeDocument(Document): class ElaspedTimeDocumentManager(): @staticmethod - def create(url, time_taken, log_datetime, type, useragent, ip): + def create(url, time_taken, log_datetime, useragent, ip): # if not hasattr(ElaspedTimeDocumentManager, 'mapping_created'): # ElapsedTimeDocument.init() # setattr(ElaspedTimeDocumentManager, 'mapping_created', True) @@ -59,7 +58,6 @@ class ElaspedTimeDocumentManager(): url=url, time_taken=time_taken, log_datetime=log_datetime, - type=type, useragent=useragent, ip=ip) doc.save() diff --git a/blog/management/commands/build_index.py b/blog/management/commands/build_index.py index d8fb12f..850b042 100644 --- a/blog/management/commands/build_index.py +++ b/blog/management/commands/build_index.py @@ -22,9 +22,8 @@ class Command(BaseCommand): help = 'build search index' def handle(self, *args, **options): + manager = ElapsedTimeDocument() + manager.init() manager = ArticleDocumentManager() manager.delete_index() manager.rebuild() - - manager = ElapsedTimeDocument() - manager.init() diff --git a/blog/middleware.py b/blog/middleware.py index a73fac2..4732baf 100644 --- a/blog/middleware.py +++ b/blog/middleware.py @@ -44,7 +44,6 @@ class OnlineMiddleware(object): url=url, time_taken=time_taken, log_datetime=timezone.now(), - type='blog', useragent=http_user_agent, ip=ip) response.content = response.content.replace( From eb0e84bd87b48bd5b9013d471a3e502e9c076d93 Mon Sep 17 00:00:00 2001 From: liangliang Date: Fri, 14 May 2021 20:09:11 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- blog/documents.py | 5 +++++ blog/management/commands/build_index.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/blog/documents.py b/blog/documents.py index 745b3b7..f8b25bb 100644 --- a/blog/documents.py +++ b/blog/documents.py @@ -43,6 +43,11 @@ class ElapsedTimeDocument(Document): class ElaspedTimeDocumentManager(): + @staticmethod + def delete_index(): + from elasticsearch import Elasticsearch + es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) + es.indices.delete(index='performance', ignore=[400, 404]) @staticmethod def create(url, time_taken, log_datetime, useragent, ip): diff --git a/blog/management/commands/build_index.py b/blog/management/commands/build_index.py index 850b042..39e4bce 100644 --- a/blog/management/commands/build_index.py +++ b/blog/management/commands/build_index.py @@ -11,7 +11,7 @@ @time: 2019-04-20 20:39 """ -from blog.documents import ElapsedTimeDocument, ArticleDocumentManager +from blog.documents import ElapsedTimeDocument, ArticleDocumentManager, ElaspedTimeDocumentManager from django.core.management.base import BaseCommand from blog.models import Article @@ -23,6 +23,7 @@ class Command(BaseCommand): def handle(self, *args, **options): manager = ElapsedTimeDocument() + ElaspedTimeDocumentManager.delete_index() manager.init() manager = ArticleDocumentManager() manager.delete_index() From 75f1081f37e88ed31f19dad25c4f8ee20c5dbd4e Mon Sep 17 00:00:00 2001 From: liangliang Date: Fri, 14 May 2021 20:15:16 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index d9914ea..68f1d0b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,7 +7,6 @@ omit = *.html *whoosh_cn_backend* *apps* - *commands* *oauth* *travis_test* *settings.py*