forked from p46318075/CodePattern
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
44 lines
1.3 KiB
44 lines
1.3 KiB
'''
|
|
使用 multiprocessing.Manager:
|
|
Manager 提供了一个可以在不同进程之间共享和修改的数据类型，如 list, dict, Namespace 等。
|
|
它实际上是在背后启动了一个单独的服务器进程，其他进程通过代理来访问这些共享对象。
|
|
'''
|
|
# 使用 multiprocessing.Manager 来完成统计词频
|
|
# 用消费者模式更好
|
|
|
|
from cppy.cp_util import *
|
|
from collections import Counter
|
|
from multiprocessing import Manager, Process
|
|
|
|
stop_words = get_stopwords()
|
|
|
|
def count_words(chunk,word_count):
|
|
words = [ w for w in chunk if ( not w in stop_words ) and len(w) >= 3 ]
|
|
for word in words:
|
|
word_count[word] = word_count.get(word, 0) + 1
|
|
# word_count.update( Counter(words) ) # 类型不起作用
|
|
|
|
@timing_decorator
|
|
def main():
|
|
manager = Manager()
|
|
word_count = manager.dict()
|
|
|
|
chunks = get_chunks(testfilepath)
|
|
processes = []
|
|
for chunk in chunks:
|
|
p = Process(target=count_words, args=(chunk,word_count))
|
|
processes.append(p)
|
|
p.start()
|
|
|
|
for p in processes: p.join()
|
|
|
|
sorted_word_count = sorted(word_count.items(), key=lambda x: x[1], reverse=True)
|
|
top_10_words = sorted_word_count[:10]
|
|
|
|
print("频率最高的10个词:")
|
|
for word, count in top_10_words:
|
|
print(f"{word}: {count}")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|