const fs = require('fs');
const lodash = require('lodash');
const csv = require('fast-csv');

// 读取CSV文件，跳过有问题的行并给出警告
csv.parseFile('chat.txt', { headers: true }, (err, data) => {
    if (err) throw err;
    const keyword = 'ai';
    const keywordData = data.filter(item => item.chat.includes(keyword));
    const wordCounts = lodash.countBy(data, 'chat');
    const top8Common = lodash.takeOrdered(wordCounts, 8, lodash.identity);
    
    const stats = {
        Keyword: keyword,
        Count: keywordData.length,
    };
    
    const results = {
        stats: [stats],
        top8: top8Common,
        keywordData: keywordData,
    };
    
    fs.writeFileSync('statistics.json', JSON.stringify(results, null, 2), 'utf-8');
});