Word frequency statistics

FileProcess.py

@@ -0,0 +1,92 @@
+#文件的处理
+import os
+import re
+from collections import Counter
+from os import listdir
+from os.path import join, isfile, isdir
+import jieba
+
+filter=[".txt"] #设置过滤后的文件类型 当然可以设置多个类型
+
+#查找当前文件路径下的所有文本文件
+def all_path(dirname):
+    print(dirname)
+    xf = 0
+    adict = {}
+    for subpath in listdir(dirname):
+        path = join(dirname, subpath)
+        if isfile(path):
+            ext = os.path.splitext(path)[1]
+            if ext in filter: #筛选文件中以.txt结尾的文件
+                xf = xf+1
+                path = path.replace(dirname, "")
+                path = path.lstrip(path[0:1])
+                adict[xf] = path
+        elif isdir(path):   #跳过文件夹，只查找目录下的文件
+            continue
+    for key, value in adict.items():
+        print(str(key)+"  :  "+value)
+    return adict
+
+#处理中文文件
+def Copenfile(name):
+    strcloud = ""
+    cut_words = ""
+    for line in open(name, encoding='utf-8'):
+        line.strip('\n')
+        line = re.sub("[A-Za-z0-9\：\·\?\!\？\；\、\—\，\。\“ \”]", "", line)  # 去除杂乱字符
+        seg_list = jieba.cut(line, cut_all=False)
+        cut_words += (" ".join(seg_list))
+    all_words = cut_words.split()
+    #将分词结果，以空格为间隔形成字符串，方便进行词云的生成
+    for s in range(0, len(all_words)):
+        strcloud = strcloud + all_words[s] + ' '
+    c = Counter()
+    for x in all_words:
+        if len(x) > 1 and x != '\r\n':  # 长度大于一，并且不为换行等字符
+            c[x] += 1
+    #新建文件存放词频统计结果
+    resfile = "res" + name
+    f = open(resfile, 'w', encoding='utf-8')
+    print("-----------------------统计词频中--------------------------")
+    print('词频统计结果：')
+    for (k, v) in c.most_common(len(all_words)):
+        print("%s:%d" % (k, v))
+        f.write(k + ':' + str(v) + '\n')#将结果存入文件
+    f.close() #关闭文件
+    return strcloud
+
+
+#处理英文文件
+def Eopenfile(name):
+    strcloud = ""
+    dic = {}#设置字典，存放结果
+    allword = []
+    for line in open(name, encoding='utf-8'):
+        line.lower()#转换为小写字母
+        newline = re.split('[ ,.?!;:"]', line.strip('\n'))
+        #将分词结果，以空格为间隔形成字符串，方便进行词云的生成
+        for s in range(0, len(newline)):
+            strcloud = strcloud + newline[s] + ' '
+        #统计单词出现次数
+        for i in newline:
+            if i in dic:
+                dic[i] += 1
+            else:
+                dic[i] = 1
+        allword +=newline
+    # 新建文件存放词频统计结果
+    resfile = "res" + name
+    f = open(resfile, 'w', encoding='utf-8')
+    # 为便于排序，将词典转化为列表
+    items = list(dic.items())
+    # 根据单词的频数从高到低排序
+    items.sort(key=lambda x: x[1], reverse=True)
+    print("-----------------------统计词频中--------------------------")
+    print('词频统计结果：')
+    for i in range(len(items)):
+        print(items[i])
+        f.write(str(items[i]) + "\n")#将结果存入文件
+    f.close()
+    return strcloud
+