diff --git a/.coveragerc b/.coveragerc index d9914ea..68f1d0b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,7 +7,6 @@ omit = *.html *whoosh_cn_backend* *apps* - *commands* *oauth* *travis_test* *settings.py* diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 6f66747..afa3c0e 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -13,44 +13,57 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [ 3.6, 3.7, 3.8, 3.9 ] steps: - - name: Start MySQL - uses: samin/mysql-action@v1.3 - with: - # The port of host - host port: 3306 # optional, default is 3306 - # The port of container - container port: 3306 # optional, default is 3306 - # --character-set-server - The character set of MySQL server - character set server: utf8mb4 # optional, default is utf8mb4 - # --collation-server - The character collation of MySQL server - collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci - # Version of MySQL to use - mysql version: latest # optional, default is latest - # MYSQL_ROOT_PASSWORD - root superuser password - mysql root password: root # optional, default is - # MYSQL_DATABASE - name for the default database that is created - mysql database: djangoblog # optional, default is - # MYSQL_USER - create the specified user with superuser power for created database - mysql user: root # optional, default is - # MYSQL_PASSWORD - specified superuser password which user is power for created database - mysql password: root # optional, default is - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Run Tests - env: - DJANGO_MYSQL_PASSWORD: root - DJANGO_MYSQL_HOST: 127.0.0.1 - run: | - python manage.py makemigrations - python manage.py migrate - python manage.py test + - name: Start MySQL + uses: samin/mysql-action@v1.3 + with: + # The port of host + host port: 3306 # optional, default is 3306 + # The port of container + container port: 3306 # optional, default is 3306 + # --character-set-server - The character set of MySQL server + character set server: utf8mb4 # optional, default is utf8mb4 + # --collation-server - The character collation of MySQL server + collation server: utf8mb4_general_ci # optional, default is utf8mb4_general_ci + # Version of MySQL to use + mysql version: latest # optional, default is latest + # MYSQL_ROOT_PASSWORD - root superuser password + mysql root password: root # optional, default is + # MYSQL_DATABASE - name for the default database that is created + mysql database: djangoblog # optional, default is + # MYSQL_USER - create the specified user with superuser power for created database + mysql user: root # optional, default is + # MYSQL_PASSWORD - specified superuser password which user is power for created database + mysql password: root # optional, default is + - name: Configure sysctl limits + run: | + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + + - uses: miyataka/elasticsearch-github-actions@1 + with: + stack-version: '7.12.1' + plugins: 'https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.12.1/elasticsearch-analysis-ik-7.12.1.zip' + + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run Tests + env: + DJANGO_MYSQL_PASSWORD: root + DJANGO_MYSQL_HOST: 127.0.0.1 + DJANGO_ELASTICSEARCH_HOST: 127.0.0.1:9200 + run: | + python manage.py makemigrations + python manage.py migrate + python manage.py test diff --git a/DjangoBlog/settings.py b/DjangoBlog/settings.py index 750c2e9..8623787 100644 --- a/DjangoBlog/settings.py +++ b/DjangoBlog/settings.py @@ -195,7 +195,7 @@ CACHES = { SITE_ID = 1 BAIDU_NOTIFY_URL = os.environ.get('DJANGO_BAIDU_NOTIFY_URL') \ - or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn' + or 'http://data.zz.baidu.com/urls?site=https://www.lylinux.net&token=1uAOGrMsUm5syDGn' # Email: EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' @@ -293,3 +293,12 @@ COMPRESS_JS_FILTERS = [ MEDIA_ROOT = os.path.join(BASE_DIR, 'uploads') MEDIA_URL = '/media/' X_FRAME_OPTIONS = 'SAMEORIGIN' + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +if os.environ.get('DJANGO_ELASTICSEARCH_HOST'): + ELASTICSEARCH_DSL = { + 'default': { + 'hosts': os.environ.get('DJANGO_ELASTICSEARCH_HOST') + }, + } diff --git a/blog/documents.py b/blog/documents.py index 9fb15b6..f8b25bb 100644 --- a/blog/documents.py +++ b/blog/documents.py @@ -28,8 +28,7 @@ class ElapsedTimeDocument(Document): url = Text() time_taken = Long() log_datetime = Date() - type = Text(analyzer='ik_max_word') - useragent = Text() + useragent = Text(analyzer='ik_max_word', search_analyzer='ik_smart') ip = Text() class Index: @@ -44,9 +43,14 @@ class ElapsedTimeDocument(Document): class ElaspedTimeDocumentManager(): + @staticmethod + def delete_index(): + from elasticsearch import Elasticsearch + es = Elasticsearch(settings.ELASTICSEARCH_DSL['default']['hosts']) + es.indices.delete(index='performance', ignore=[400, 404]) @staticmethod - def create(url, time_taken, log_datetime, type, useragent, ip): + def create(url, time_taken, log_datetime, useragent, ip): # if not hasattr(ElaspedTimeDocumentManager, 'mapping_created'): # ElapsedTimeDocument.init() # setattr(ElaspedTimeDocumentManager, 'mapping_created', True) @@ -59,7 +63,6 @@ class ElaspedTimeDocumentManager(): url=url, time_taken=time_taken, log_datetime=log_datetime, - type=type, useragent=useragent, ip=ip) doc.save() diff --git a/blog/management/commands/build_index.py b/blog/management/commands/build_index.py index d8fb12f..39e4bce 100644 --- a/blog/management/commands/build_index.py +++ b/blog/management/commands/build_index.py @@ -11,7 +11,7 @@ @time: 2019-04-20 20:39 """ -from blog.documents import ElapsedTimeDocument, ArticleDocumentManager +from blog.documents import ElapsedTimeDocument, ArticleDocumentManager, ElaspedTimeDocumentManager from django.core.management.base import BaseCommand from blog.models import Article @@ -22,9 +22,9 @@ class Command(BaseCommand): help = 'build search index' def handle(self, *args, **options): + manager = ElapsedTimeDocument() + ElaspedTimeDocumentManager.delete_index() + manager.init() manager = ArticleDocumentManager() manager.delete_index() manager.rebuild() - - manager = ElapsedTimeDocument() - manager.init() diff --git a/blog/middleware.py b/blog/middleware.py index a73fac2..4732baf 100644 --- a/blog/middleware.py +++ b/blog/middleware.py @@ -44,7 +44,6 @@ class OnlineMiddleware(object): url=url, time_taken=time_taken, log_datetime=timezone.now(), - type='blog', useragent=http_user_agent, ip=ip) response.content = response.content.replace( diff --git a/blog/tests.py b/blog/tests.py index ebc92f3..74cc9e3 100644 --- a/blog/tests.py +++ b/blog/tests.py @@ -199,12 +199,11 @@ class ArticleTest(TestCase): self.assertEqual(rsp.status_code, 404) def test_commands(self): + from blog.documents import ELASTICSEARCH_ENABLED + if ELASTICSEARCH_ENABLED: + call_command("build_index") call_command("ping_baidu", "all") call_command("create_testdata") call_command("clear_cache") call_command("sync_user_avatar") call_command("build_search_words") - - from blog.documents import ELASTICSEARCH_ENABLED - if ELASTICSEARCH_ENABLED: - call_command("build_index") diff --git a/requirements.txt b/requirements.txt index b3b51ae..55d2937 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,42 +1,40 @@ -coverage==5.4 -Django==3.1.8 +coverage==5.5 +Django==3.2.2 django-appconf==1.0.4 django-autoslug==1.9.8 -django-compressor==2.4 +django-compressor==2.4.1 django-debug-toolbar==3.2.1 django-haystack==3.0 django-ipware==3.0.2 django-mdeditor==0.1.18 django-uuslug==1.2.0 -elasticsearch==7.11.0 +elasticsearch==7.12.1 elasticsearch-dsl==7.3.0 gevent==21.1.2 greenlet==0.4.17 -importlib-metadata==3.7.0 ipaddress==1.0.23 -isort==5.7.0 +isort==5.8.0 jieba==0.42.1 jsonpickle==2.0.0 -lazy-object-proxy==1.5.2 +lazy-object-proxy==1.6.0 markdown2==2.4.0 mccabe==0.6.1 mistune==0.8.4 mysqlclient==2.0.3 -Pillow==8.1.1 +Pillow==8.2.0 pycparser==2.20 -Pygments==2.8.0 -pylint==2.7.1 +Pygments==2.9.0 +pylint==2.8.2 pyparsing==2.4.7 python-dateutil==2.8.1 python-logstash==0.4.6 python-memcached==1.59 -python-slugify==4.0.1 +python-slugify==5.0.2 pytz==2021.1 raven==6.10.0 rcssmin==1.0.6 requests==2.25.1 rjsmin==1.1.0 -six==1.15.0 urllib3==1.26.4 WeRoBot==1.13.1 Whoosh==2.7.4 \ No newline at end of file