From c0b1aa58235fd4894976a35ff6382211c8d6d760 Mon Sep 17 00:00:00 2001
From: qweasdzxc227 <1095578930@qq.com>
Date: Tue, 4 Jun 2024 13:46:24 +0800
Subject: [PATCH] 6.4.13.46

---
 .../__pycache__/items.cpython-39.pyc          | Bin 2756 -> 2800 bytes
 .../__pycache__/pipelines.cpython-39.pyc      | Bin 6549 -> 6114 bytes
 .../__pycache__/settings.cpython-39.pyc       | Bin 1211 -> 1211 bytes
 ArticleSpider/ArticleSpider/items.py          |   2 ++
 .../__pycache__/es_types.cpython-39.pyc       | Bin 1649 -> 1648 bytes
 .../ArticleSpider/models/es_types.py          |   1 +
 ArticleSpider/ArticleSpider/pipelines.py      |  10 ++++++----
 ArticleSpider/ArticleSpider/settings.py       |   4 ++--
 .../__pycache__/jobbole.cpython-39.pyc        | Bin 3643 -> 3716 bytes
 .../ArticleSpider/spiders/jobbole.py          |   3 ++-
 10 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/ArticleSpider/ArticleSpider/__pycache__/items.cpython-39.pyc b/ArticleSpider/ArticleSpider/__pycache__/items.cpython-39.pyc
index f34a24f54087e2a87ab795e690d80c891f845ffb..8618b9ab53ce685cce26cb139db62e2fdc60d41c 100644
GIT binary patch
delta 243
zcmX>i`azU0k(ZZ?0SIho#ie=jY~(w_%GfjcGHbO!D)R!Cg$z+5DXb}MEeuhjlf&4u
z8Fx%R#MWpN&r`#Y!XwF0!<fYz&sV}<!jL79B?#m(rtqdQ0cq|OzBHyZre+{JUZ{p~
zf$%~GMurrIU<OV8$))T?YFy<RsYR*rX+`<Dw|Gm7a^mxovQm>v;xkiTf*kXb9Y|=3
zOlIV;VJdQ*Y{e1F<_;7oR+>DQLw52G4oya}$&WZJ7@a0da~iTa0;P(4Ci`+KGOA6^
f=TsA9<X|c?0+}FLqy>`moxFr|6Qld&aIRbczB)c)

delta 207
zcmew$dPI~jk(ZZ?0SKl|4^4Z^y^-$-D`U&#%dFKLDXb}MEeug2ljGR38P`od#nxz)
z#S_n4!dJqO#h)cm!<faG!jr}Xq`6ag)0onjnt|+i!5YQ|LJJuf8B!R688rDOx3d>X
z@s<|l#OEhvr6!leXQsRaS@)71NN5UAmglgUoWK#q<_6>zD@;DXAv^g#hdQI^WOhyq
yM#sqxoQ90vlh<*HPwwO7VpN$tlT(eKsmKVVSF%V8NEP`^zQDPO(QR@wS1thMDK>rp

diff --git a/ArticleSpider/ArticleSpider/__pycache__/pipelines.cpython-39.pyc b/ArticleSpider/ArticleSpider/__pycache__/pipelines.cpython-39.pyc
index 983cf6541cdd9c9a3907a4aabb183dc8876dc5ed..bb491947f1b4cf6889880a5bd40a2ab849091f79 100644
GIT binary patch
delta 712
zcmYk2&ubGw6vy)>yPNGM+1>o!G^tHe!7ddlCe~UL5y1vStCXsd9Fm46vGGSzC!!KP
z6!cJg3(tZF#eyPwEA$@_PkPgfLa_(Ed+ebH!Fe{QJIrU_eCN%Z@0<1Xk8!=~^=c0B
zU3*?`I6^%=4&QoXU)+I*&Qi1{&pYh!Rvb<{nFp{e?YN~4cmNx$=Bqg|BOj9d=c|)g
zk>Bcv6L**Dyk2SW_Dx&E&)!pbM3K<Y&)}M}?%CG<ZR4<O6%OJX*T~jKR{$V~J8A@`
zwszGVEz(*tnG=pWVe-WoDE+}ej6G$*gl-aHAc&8(qR9WQ?ZOOxbr+ZeaKy9kk@Y?9
zd1B;o7A^0l=YNm*!JF?pe3Ifv#Z9)t)8fZ7gkyvW!g0bRzSBdaCn%gEoFsVu!bed-
z;G&qF1pd+!7&G$bInw=v8A1VvEi0B4Ayw+oc@<PA1I??-s>HLHj||}p{}}!>UP23d
z=2K;w4i<6THvzY>;cG$>Klw6n2W9^SSfGNTIgxUM3My2PsZLeqbNy1U2j_)2Z8J{R
z_!R+L=9eihVkMA;`}izyJ+&x=lG$jsnw_Oe{oZn$-)^?9@*?Gx2^F-130T2Gunvp3
z8MNUMZU@go4HKcEP*tRK#oTuQf10TZ2d|7ZQY8TgsL)N}N+g7aH5#r9WtH}rfXX<n
r?LnNi+R7yn@)EwcxJggNsXb0N@VaO>f(uat61W_VyAGIXQ#bYB^iQ20

delta 1114
zcmZvbO>7%Q6vyX{*Sl-SS%2Gc-JqHZi3J2zTA`LEKwCnaRG3ghqDB<QcsH=hIySRT
z)f$@|f|PPVsCs}ZAywkmo`RH1PjKUc#1YU+y&$3-TJ?rdk&t-L4dp}ZXnynF=*|Dl
z?7aE;na>VejnUD#pzr(L+VaWLRck-oz53)2<;X^4v2<EE1*iC-aEfl0{C!TeChW28
z1iToLzYDBtw{a?ZKucE}k7FZxQ`&J%TF>L6{v%EqSKv|n);PF*$4CQA;vZ%aj&0{-
zQ|#2@mC7U~4GNrEO3?S31Zwy7SW5OPSc&EEW_*F-VdPghi@%M`%6kG&C2r%yXpYlP
z;q2&Fs9UEg=Sf8CoMG!~f+X>-ReFJJi)>Y4BpF8;#~9Bs5)YAPYliST_23bLkocWZ
z#^vO6>I``*!#K;B!Rwi^@^La*E?^N8%!q`zWkyX+9mjjgJJ7-~bxoh<1@m|*{Va5F
zEA7ENmNG|R4Xc?~q0U12ISN@}A)SS!88M@3jwH@tHv9g*b2Ry~Eme(1wLq|=>J7H%
z@kVwMdbpEafw%FsT;ZvCp1#0%i}CNng^8x$4%~Xs@f=sJBj#LMc0XswHz+ue1cBFc
zWB5}#fphsWXyUjzo?oKr%hVri4u<DjqRoxtt$cYscS3c&rti+Q8t=GsUdQ#lRkx{8
z{9m6*2A9_<>VYEkjyTYl7loQ|<Us4gOX5Q6O*x1T^uF#yN@AeBm+5PL>1dOpFBe2j
zEQr;kn&^tlK8S$<qIaMRePaVatUJ-a19fp;I27=KQQI32i9=*j#p+dMQ)=}XA=EmJ
z?z`#<GS|2ycE0#}_oK~UZhgG-$=2?tH*6WkrY|+!PS<O%s#hq$HitzN8p;jU{BAHT
zq)_u#m)q-$NsmunU-o>r&bpy&uY}Qy4S&rI!Ts+QSYFp{)tl{B$9KD~bA{9&JOC<s
z6EsLboW?j<L=$=k|L^ZU{SS>IiBxS(<w!{78T_1-&t;W%$lRnpyjl~Kvv+HoBiDF#
zcu6UoEu1K{$p8Fdq*UzNW&E<x*7-H4^H?aVJlDJ8T2)I7mRBvxRK%@flAiiE#r?7I
K)ZvsxE%_((`29Hm

diff --git a/ArticleSpider/ArticleSpider/__pycache__/settings.cpython-39.pyc b/ArticleSpider/ArticleSpider/__pycache__/settings.cpython-39.pyc
index b08b48758091daa40111f2c0d22fa9b3c7b3f8b5..93df99d4d7d2f9a726460eb8c786766d4d9662ce 100644
GIT binary patch
delta 20
acmdnZxto(ak(ZZ?0SHP~#%<)@zybg=kp%<*

delta 20
acmdnZxto(ak(ZZ?0SJU9hi&BEzybg-Hw1(L

diff --git a/ArticleSpider/ArticleSpider/items.py b/ArticleSpider/ArticleSpider/items.py
index 5076fd8..adfce61 100644
--- a/ArticleSpider/ArticleSpider/items.py
+++ b/ArticleSpider/ArticleSpider/items.py
@@ -66,6 +66,7 @@ class JobBoleArticleItem(scrapy.Item):
         output_processor=Join(separator=',')
     )  # 标签
     content = scrapy.Field()  # 内容
+    where_from = scrapy.Field()  # 来自
 
     def save_to_es(self):
         article = ArticleType()
@@ -80,6 +81,7 @@ class JobBoleArticleItem(scrapy.Item):
         article.comment_nums = self['comment_nums']
         article.url = self['url']
         article.tags = self['tags']
+        article.where_from = self['where_from']
         article.meta.id = self['url_object_id']
         article.suggest = gen_suggests(ArticleType._doc_type.index, ((article.title, 10), (article.tags, 7)))
         article.save()
diff --git a/ArticleSpider/ArticleSpider/models/__pycache__/es_types.cpython-39.pyc b/ArticleSpider/ArticleSpider/models/__pycache__/es_types.cpython-39.pyc
index 482af49fcf38fd40c72dd2148a0ece2e4fb0bfdf..41161e01826c4a3c8fba60c7c969ac60f500449b 100644
GIT binary patch
delta 674
zcmZuu&ui2`7|l1CWYZ**&32pJDppSlD+mP_yoe$mJlTsqMKFZSbd622Ws;>-1P>K&
zdsyZlXi@(HFW&qwq~y|*cfEP^n^f=zCd}h~-^+aOW#+s8&1WfNu7S_j(`S1hS8VIj
zdn~xYp)g7#Hb8SyKuO}&!|O#?nuTAoIOsSn`ehi0V2JR=fLk-*_7cE&Wbht$-Wc2w
z!Lb=L!T1_?QT3K2UPsmE3{@rwszb5CgMA`T#PB3K)t3Dy#w1*wp952$;kG@%NOGWl
zz*Fd}T@v5i=o){+P?roLz*Q3j2P9Kx<UYJon`Sl@HYXgG5J%{5a*F=+n8YJ)p_|=$
z4`h>Ph1fkh5EGC+J#JHdFcUA*bfjUd`ZRid5l7b@)5a}*ESmHNvYH||KtWzbvNIXW
zKel5>t|7EuzMrI3S{%I;GVaxj*-X@p^t5C(c~c0f$FyE1McRmjBS?Kst+|lu9i4)&
z&gh-2bq}8lDUv-oFFU`BM85-ehjJM=>iG2)1c$ibg8k2{doGds%hpMfGR><b*_n(a
zQ3uwTn%Gxwty}7owS{-`#hMCz%I)A+zHDAqB5U$_C9h$>o74SwOY<sk{+FtC?v``D
YD8xNIyk1A}VcYbHPXaTva77OM1cX6~Q2+n{

delta 668
zcmZuuyN=W_6pfvDHanAKCc``!9%7Mpk+Pyd0*L~&2vHykwpl7uSee)iVe<edBQZ2v
zEbSB{chG>e1s{O;3+j;!KR|&5UjWyEW~H^{qjT@E?>)Z$<^A^j*!SBOtUuolp1oW7
zmnWYf;1;zw;y6P&q6isoXR8N0N4ZD6CUi1C4`{dHB<tlN4Ur|n7k%EEA-=JMOAjsX
z^3Asv-xS?rn+77_ZQcRZTVi?+RG$Z+2Eqn)Acnkqh~<eGd>DPSpl<KLib74M<#f9E
zVH`$CufDHQpgyCUtv)1VU;RW+QBOU>ba&V!PQ$>20YDdag#lb71N9Z(LsI#6vI{0`
z?tlsB7}Fsq(6XgF_hp?XnRv2zDIz3YQ)Hq}Y<4s<bZp=}fF^0d#cS&FGGR3wmj6r{
zy<w)kb{>m5z6Pvv09@oCc?HORG?J&wN%IZ5lOD3Th_l5TA?ZdnpUp&7OV3DTQm1t$
zBx%kiQpB}j96)x|d*=@F)!Mne-BdrziM$B+jQQ$<+9Q{);wGV8#_~8V7~7A=y3JTT
zub-EaF=?w;WUQ9tI+nirLbhP=YjSI5MybR9-IKB?L{g_^A-5pkRBWE5owP{nvutSh
dZeH?vCicvKRAxpF_RtO6!yXRo9)Ue@(O<dMlZ^lX

diff --git a/ArticleSpider/ArticleSpider/models/es_types.py b/ArticleSpider/ArticleSpider/models/es_types.py
index 5fd480d..1eaa740 100644
--- a/ArticleSpider/ArticleSpider/models/es_types.py
+++ b/ArticleSpider/ArticleSpider/models/es_types.py
@@ -33,6 +33,7 @@ class ArticleType(DocType):
     fav_nums = Integer()
     tags = Text(analyzer="ik_max_word")
     content = Text(analyzer="ik_max_word")
+    where_from = Text(analyzer="ik_max_word")
 
     class Meta:
         index = "jobbole"
diff --git a/ArticleSpider/ArticleSpider/pipelines.py b/ArticleSpider/ArticleSpider/pipelines.py
index 425e4d6..68edf95 100644
--- a/ArticleSpider/ArticleSpider/pipelines.py
+++ b/ArticleSpider/ArticleSpider/pipelines.py
@@ -32,8 +32,8 @@ class MysqlPipeline(object):
 
     def process_item(self, item, spider):
         insert_sql = """
-            insert into jobbole_article(title, url ,url_object_id, front_image_url, front_image_path, parise_nums, comment_nums, fav_nums, tags, content, create_date)
-            values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE parise_nums=VALUES(parise_nums)
+            insert into jobbole_article(title, url ,url_object_id, front_image_url, front_image_path, parise_nums, comment_nums, fav_nums, tags, content, create_date, where_from)
+            values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE parise_nums=VALUES(parise_nums)
         """
         params = list()
         params.append(item.get("title", ""))
@@ -48,6 +48,7 @@ class MysqlPipeline(object):
         params.append(item.get("tags", ""))
         params.append(item.get("content", ""))
         params.append(item.get("create_date", "1970-07-01"))
+        params.append(item.get("where_from", ""))
         self.cursor.execute(insert_sql, tuple(params))
         self.conn.commit()
         return item
@@ -83,8 +84,8 @@ class MysqlTwistedPipline(object):
     def do_insert(self, cursor, item):
         # 执行具体的插入
         insert_sql = """
-                   insert into jobbole_article(title, url ,url_object_id, front_image_url, front_image_path, parise_nums, comment_nums, fav_nums, tags, content, create_date)
-                   values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE parise_nums=VALUES(parise_nums)
+                   insert into jobbole_article(title, url ,url_object_id, front_image_url, front_image_path, parise_nums, comment_nums, fav_nums, tags, content, create_date, where_from)
+                   values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE parise_nums=VALUES(parise_nums)
                """
         params = list()
         params.append(item.get("title", ""))
@@ -99,6 +100,7 @@ class MysqlTwistedPipline(object):
         params.append(item.get("tags", ""))
         params.append(item.get("content", ""))
         params.append(item.get("create_date", "1970-07-01"))
+        params.append(item.get("where_from", ""))
         # 根据不同的item 构建不同的sql语句并插入到mysql中
         cursor.execute(insert_sql, tuple(params))
 
diff --git a/ArticleSpider/ArticleSpider/settings.py b/ArticleSpider/ArticleSpider/settings.py
index abdc2b1..8585f5e 100644
--- a/ArticleSpider/ArticleSpider/settings.py
+++ b/ArticleSpider/ArticleSpider/settings.py
@@ -74,8 +74,8 @@ ITEM_PIPELINES = {
     # 'ArticleSpider.pipelines.JsonWithEncodingPipeline': 2,
     # 'ArticleSpider.pipelines.JsonExporterPipeline': 3,
     # 'ArticleSpider.pipelines.MysqlPipeline': 4,
-    # 'ArticleSpider.pipelines.MysqlTwistedPipline': 5,
-    'ArticleSpider.pipelines.ElasticsearchPipeline': 6,
+    'ArticleSpider.pipelines.MysqlTwistedPipline': 5,
+    # 'ArticleSpider.pipelines.ElasticsearchPipeline': 6,
     'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
 }
 
diff --git a/ArticleSpider/ArticleSpider/spiders/__pycache__/jobbole.cpython-39.pyc b/ArticleSpider/ArticleSpider/spiders/__pycache__/jobbole.cpython-39.pyc
index 639c1c684188b942062f2c23ea5b1dd90c33385b..514c62de25f81a50d61401151f4ef3cb04f79577 100644
GIT binary patch
delta 368
zcmdlj(;~~4$ji&c00fTn;?hnEZsfbk#>lw&1)C|Oq#01OSO_G?EX2scD8eWJ#BBe$
zm?s-?7>exxikC1fU`%1GVa#G$$W+Tz!&J0oawUgfeG2bFCPs!DmIWLsd=N<}j~~e6
zT)>sWpCYgjsD+WCgu8@i0WU~Rir@mi8s>$J3z=$Iz<i-OOf?Ky{3*h{%wY_S3@Hr3
z44NW-RUCR+_8KuM+L}dLKnG|t-D1fsNzE-X26Aq(mSmRXq*hI~;>^~$#Z{h>T9g`}
zR+OJxC8w{i7449jVyl#snU@`3oL^d$oNA;Lt6!2@QKF$ac`0WcqrhYqE-mgT&cu|I
z_=<wWl8niETosDWKy!;_fCK|0A0rDR7m(zELq0|^#?42$x*5fTK*|I_%9wb7N(8`A
VhH-K_&s<g+4qgua&78bOi~w?GQoH~F

delta 269
zcmZpX-7UkH$ji&c00b-lgr;4W+{ky6jWK2O3pP_mQ7fQmF&{{dS%{H?QG`)|QH5!;
z8Hb_R8lYGS!ve+>#u~;friDzkOf^hJD<;=+_(ekm7#V6<7O<!A0(l$@I8%62_!a_{
zF*1~Jm2fZM0jWvhU%*?#ypVArQw<B4FEEFxh9QeDMX;ASjDe9Mg&~+hQ^>E1Lr=?I
zBPK;#^CjbCBhKc@4>{u)`6oMYX-zKWs*raBIZ+DaL`FVF7Dg^nK1LoW<YN?L+swz^
o%_v?7QX~LY#KXwPA^`M<0HX}+<PAJ?Srs^VIJ7qh@ftA#0Kt_uj{pDw

diff --git a/ArticleSpider/ArticleSpider/spiders/jobbole.py b/ArticleSpider/ArticleSpider/spiders/jobbole.py
index 56c5adb..3912247 100644
--- a/ArticleSpider/ArticleSpider/spiders/jobbole.py
+++ b/ArticleSpider/ArticleSpider/spiders/jobbole.py
@@ -54,7 +54,7 @@ class JobboleSpider(scrapy.Spider):
     def parse(self, response):
         # 1.获取新闻列表页中的新闻url并交给scrapy进行下载后调用相应的解析方
         # 提取文章链接,extract_first()提取第一个值
-        post_nodes = response.css('#news_list .news_block')[:100]
+        post_nodes = response.css('#news_list .news_block')[:1]
         for post_node in post_nodes:
             image_url = "https:" + post_node.css('.entry_summary a img::attr(src)').extract_first("")
             post_url = post_node.css('h2 a::attr(href)').extract_first("")
@@ -73,6 +73,7 @@ class JobboleSpider(scrapy.Spider):
             item_loader.add_css('content', '#news_content')
             item_loader.add_css('tags', '.news_tags a::text')
             item_loader.add_css('create_date', '#news_info .time::text')
+            item_loader.add_xpath('where_from', '//*[@id="link_source2"]/text()')
             item_loader.add_value('url', response.url)
             item_loader.add_value('front_image_url', response.meta.get('front_image_url', ''))
             yield Request(url=parse.urljoin(response.url, "/NewsAjax/GetAjaxNewsInfo?contentId={}".format(post_id)),