You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
105 lines
3.2 KiB
105 lines
3.2 KiB
9 months ago
|
import sys, re, operator, string
|
||
|
|
from cppy.cp_util import *
|
||
|
|
|
||
|
|
class DataStorageManager():
|
||
|
|
""" Models the contents of the file """
|
||
|
|
def __init__(self):
|
||
|
|
self._data = []
|
||
|
|
|
||
|
|
def dispatch(self, message):
|
||
|
|
if message[0] == 'init':
|
||
|
|
return self._init(message[1])
|
||
|
|
elif message[0] == 'words':
|
||
|
|
return self._words()
|
||
|
|
else:
|
||
|
|
raise Exception("Message not understood " + message[0])
|
||
|
|
# 使用内省的写法
|
||
|
|
'''
|
||
|
|
def dispatch(self, message):
|
||
|
|
method_name = '_' + message[0]
|
||
|
|
if hasattr(self, method_name):
|
||
|
|
method = getattr(self, method_name)
|
||
|
|
return method(*message[1:])
|
||
|
|
else:
|
||
|
|
raise ValueError(f"DataStorageManager doesn't understand message {message[0]}")
|
||
|
|
'''
|
||
|
|
|
||
|
|
def _init(self, path_to_file):
|
||
|
|
self._data = re.findall('\w+', read_file(path_to_file).lower())
|
||
|
|
|
||
|
|
def _words(self):
|
||
|
|
return self._data
|
||
|
|
|
||
|
|
|
||
|
|
class StopWordManager():
|
||
|
|
""" Models the stop word filter """
|
||
|
|
_stop_words = []
|
||
|
|
|
||
|
|
def dispatch(self, message):
|
||
|
|
if message[0] == 'init':
|
||
|
|
return self._init()
|
||
|
|
elif message[0] == 'is_stop_word':
|
||
|
|
return self._is_stop_word(message[1])
|
||
|
|
else:
|
||
|
|
raise Exception("Message not understood " + message[0])
|
||
|
|
|
||
|
|
def _init(self):
|
||
|
|
self._stop_words = get_stopwords()
|
||
|
|
|
||
|
|
def _is_stop_word(self, word):
|
||
|
|
return word in self._stop_words
|
||
|
|
|
||
|
|
|
||
|
|
class WordFrequencyManager():
|
||
|
|
""" Keeps the word frequency data """
|
||
|
|
_word_freqs = {}
|
||
|
|
|
||
|
|
def dispatch(self, message):
|
||
|
|
if message[0] == 'increment_count':
|
||
|
|
return self._increment_count(message[1])
|
||
|
|
elif message[0] == 'sorted':
|
||
|
|
return self._sorted()
|
||
|
|
else:
|
||
|
|
raise Exception("Message not understood " + message[0])
|
||
|
|
|
||
|
|
def _increment_count(self, word):
|
||
|
|
if word in self._word_freqs:
|
||
|
|
self._word_freqs[word] += 1
|
||
|
|
else:
|
||
|
|
self._word_freqs[word] = 1
|
||
|
|
|
||
|
|
def _sorted(self):
|
||
|
|
return sorted(self._word_freqs.items(), key=operator.itemgetter(1), reverse=True)
|
||
|
|
|
||
|
|
class WordFrequencyController():
|
||
|
|
|
||
|
|
def dispatch(self, message):
|
||
|
|
if message[0] == 'init':
|
||
|
|
return self._init(message[1])
|
||
|
|
elif message[0] == 'run':
|
||
|
|
return self._run()
|
||
|
|
else:
|
||
|
|
raise Exception("Message not understood " + message[0])
|
||
|
|
|
||
|
|
def _init(self, path_to_file):
|
||
|
|
self._storage_manager = DataStorageManager()
|
||
|
|
self._stop_word_manager = StopWordManager()
|
||
|
|
self._word_freq_manager = WordFrequencyManager()
|
||
|
|
self._storage_manager.dispatch(['init', path_to_file])
|
||
|
|
self._stop_word_manager.dispatch(['init'])
|
||
|
|
|
||
|
|
def _run(self):
|
||
|
|
for w in self._storage_manager.dispatch(['words']):
|
||
|
|
if not self._stop_word_manager.dispatch(['is_stop_word', w]):
|
||
|
|
self._word_freq_manager.dispatch(['increment_count', w])
|
||
|
|
|
||
|
|
word_freqs = self._word_freq_manager.dispatch(['sorted'])
|
||
|
|
for (w, c) in word_freqs[0:10]:
|
||
|
|
print(w, '-', c)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
wfcontroller = WordFrequencyController()
|
||
|
|
wfcontroller.dispatch(['init', testfilepath])
|
||
|
|
wfcontroller.dispatch(['run'])
|