import os
import threading
from queue import Queue
from collections import Counter
import re

# 共享队列和词频统计器
file_queue = Queue()
word_counter = Counter()
lock = threading.Lock()  # 确保线程安全更新 Counter

# 读取文件并分词的函数
def process_file():
    while True:
        try:
            # 从队列获取文件名，非阻塞
            file_path = file_queue.get_nowait()
        except:
            break  # 队列为空，退出
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                text = f.read().lower()
                # 简单分词，移除标点
                words = re.findall(r'\b\w+\b', text)
                # 线程安全更新词频
                with lock:
                    word_counter.update(words)
        except Exception as e:
            print(f"Error processing {file_path}: {e}")
        finally:
            file_queue.task_done()

def main():
    # 获取 data 目录下所有 .txt 文件
    data_dir = 'data'
    files = [os.path.join(data_dir, f) for f in os.listdir(data_dir) if f.endswith('.txt')]
    
    # 将文件路径放入队列
    for file_path in files:
        file_queue.put(file_path)
    
    # 创建并启动多个线程
    num_threads = 4  # 可根据需要调整线程数
    threads = []
    for _ in range(num_threads):
        t = threading.Thread(target=process_file)
        t.start()
        threads.append(t)
    
    # 等待所有线程完成
    for t in threads:
        t.join()
    
    # 输出前 10 高频词
    print("Top 10 高频词:")
    for word, count in word_counter.most_common(10):
        print(f"{word}: {count}")

if __name__ == '__main__':
    main()