TextCNN/model_config.py

import os
import json
import torch
import os
import gensim
import torch
import json
import multiprocessing
import pandas as pd
import numpy as np
from time import time
from gensim.models import Word2Vec
from gensim.models.phrases import Phraser
from gensim.models.phrases import Phrases
from gensim.models import KeyedVectors


import os
import json
import pandas as pd
import numpy as np
from tqdm import tqdm
from collections import Counter
import warnings
warnings.filterwarnings("ignore")

def extract_three_cls_data(data_path,save_path):
    map_path = 'map.json'
    data = pd.read_csv(data_path, sep='\t').dropna()
    cls_data = data[(data['label'] == '童书') | (data['label'] == '工业技术') | (data['label'] == '大中专教材教辅')]
    cls_data.index = range(len(cls_data))
    print(Counter(cls_data['label']))
    print('总共 {} 个类别'.format(len(np.unique(cls_data['label']))))
    label_map = {key:index for index, key in enumerate(np.unique(cls_data['label']))}
    label_map_json = json.dumps(label_map, ensure_ascii=False, indent=3)
    if not os.path.exists(label_map_json):
        with open(map_path, 'w', encoding='utf-8') as f:
            f.write(label_map_json)
    cls_data['textcnn_label'] = cls_data['label'].map(label_map)
    with open('/data/bigfiles/52afd73b-7526-4120-b196-4d468218114a.txt', 'r', encoding='utf-8') as f:
        stopwords = f.readlines()
        stopwords = [i.strip() for i in stopwords]
    cls_data['text_seg'] = ''
    for idx,row in tqdm(cls_data.iterrows(), desc='去除停用词：', total=len(cls_data)):
        words = row['text'].split(' ')
        out_str = ''
        for word in words:
            if word not in stopwords:
                out_str += word
                out_str += ' '
        cls_data['text_seg'][idx] = out_str
    cls_data.to_csv(save_path, index=False)

def build_word2id(lists):
    maps = {}
    for item in lists:
        if item not in maps:
            maps[item] = len(maps)
    return maps


def build_data(train_data, word2id_map, max_length):
    data = train_data['text_seg']
    train_list = []
    label_list = train_data['textcnn_label']
    for line in data:
        train_word_list = line.split(' ')
        train_line_id = []
        for word in train_word_list:
            id = word2id_map[word]
            train_line_id.append(id)
        length = len(train_line_id)
        if length > max_length + 1:
            train_line_id = train_line_id[:max_length + 1]
        if length < max_length + 1:
            train_line_id.extend([word2id_map['PAD']] * (max_length - length + 1))
        train_list.append(train_line_id)
    return train_list, label_list


def filter_stopwords(data_path, save_path):
    if not os.path.exists(save_path):
        extract_three_cls_data(data_path, save_path)


def concat_all_data(train_path, dev_path, test_path):
    data_train = pd.read_csv(train_path)
    data_dev = pd.read_csv(dev_path)
    data_test = pd.read_csv(test_path)
    data = pd.concat([data_train, data_dev, data_test])
    data.index = range(len(data))
    return data


def gen_word2id(train_path, dev_path, test_path):
    data = concat_all_data(train_path, dev_path, test_path)
    word2id_path = 'word2id.json'
    id2word_path = 'id2word.json'
    if not os.path.exists(word2id_path):
        data_lines = data['text_seg']
        words_list = []
        for line in tqdm(data_lines, desc='gen word2id'):
            words = line.split(' ')
            words_list.extend(words)
        word2id = build_word2id(words_list)
        word2id['PAD'] = len(word2id)
        id2word = {word2id[w]: w for w in word2id}
        with open(word2id_path, 'w', encoding='utf-8') as f:
            f.write(json.dumps(word2id, ensure_ascii=False, indent=2))
        with open(id2word_path, 'w', encoding='utf-8') as f:
            f.write(json.dumps(id2word, ensure_ascii=False, indent=2))
    else:
        with open(word2id_path, 'r', encoding='utf-8') as f:
            word2id = json.load(f)
        with open(id2word_path, 'r', encoding='utf-8') as f:
            id2word = json.load(f)
    return word2id, id2word


def process_data(data_path, word2id, max_length):
    data = pd.read_csv(data_path)
    train_list, label_list = build_data(data, word2id, max_length)
    return train_list, label_list

def prepare_data(max_length):
    train_data_path = '/data/bigfiles/caa7144d-40ea-4b67-91b4-ca4bf2b4c04d.csv'
    train_save_path = 'train.csv'
    filter_stopwords(train_data_path, train_save_path)
    dev_data_path = '/data/bigfiles/15bf9d03-0569-4c87-88d9-f6c3b8deaf2a.csv'
    dev_save_path = 'dev.csv'
    filter_stopwords(dev_data_path, dev_save_path)
    test_data_path = '/data/bigfiles/6d0de77c-d0fd-4d15-84a0-99cfe53790b6.csv'
    test_save_path = 'test.csv'
    filter_stopwords(test_data_path, test_save_path)
    word2id, id2word = gen_word2id(train_save_path, dev_save_path, test_save_path)
    X_train, y_train = process_data(train_save_path, word2id, max_length)
    X_dev, y_dev = process_data(dev_save_path, word2id, max_length)
    X_test, y_test = process_data(test_save_path, word2id, max_length)
    return X_train,y_train, X_dev, y_dev, X_test, y_test



def load_w2v():
    train_save_path = 'train.csv'
    dev_save_path = 'dev.csv'
    test_save_path = 'test.csv'
    data = concat_all_data(train_save_path, dev_save_path, test_save_path)
    model_save_path = 'w2v_model.bin'
    vec_save_path = 'w2v_model.txt'
    if not os.path.exists(vec_save_path):
        sent = [str(row).split(' ') for row in data['text_seg']]
        phrases = Phrases(sent, min_count=5, progress_per=10000)
        bigram = Phraser(phrases)
        sentence = bigram[sent]
        cores = multiprocessing.cpu_count()
        w2v_model = Word2Vec(
            min_count=2,
            window=2,
            size=300,
            sample=6e-5,
            alpha=0.03,
            min_alpha=0.0007,
            negative=15,
            workers=cores - 1,
            iter=7)
        t0 = time()
        w2v_model.build_vocab(sentence)
        t1 = time()
        print('build vocab cost time: {}s'.format(t1 - t0))
        w2v_model.train(
            sentence,
            total_examples=w2v_model.corpus_count,
            epochs=20,
            report_delay=1
        )
        t2 = time()
        print('train w2v model cost time: {}s'.format(t2 - t1))
        w2v_model.save(model_save_path)
        w2v_model.wv.save_word2vec_format(vec_save_path, binary=False)


def get_pretrainde_w2v():
    w2v_path = 'w2v_model.txt'
    w2v_model = KeyedVectors.load_word2vec_format(w2v_path, binary=False)

    word2id_path = 'word2id.json'
    id2_word_path = 'id2word.json'
    with open(word2id_path, 'r', encoding='utf-8') as f:
        word2id = json.load(f)
    with open(id2_word_path, 'r', encoding='utf-8') as f:
        id2word = json.load(f)

    vocab_size = len(word2id)
    embedding_size = 300
    weight = torch.zeros(vocab_size, embedding_size)
    for i in range(len(w2v_model.index2word)):
        try:
            index = word2id[w2v_model.index2word[i]]
        except:
            continue
        weight[index, :] = torch.from_numpy(w2v_model.get_vector(
            id2word[str(word2id[w2v_model.index2word[i]])]))
    # print(weight)
    return weight


class ModelConfig():
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    vocab_size = 0
    num_classes = 3
    max_length = 64
    use_pretrained_w2v = False
    if use_pretrained_w2v:
        if not os.path.exists('w2v_model.txt'):
            load_w2v()
        embedding_pretrained = get_pretrainde_w2v()
    else:
        embedding_pretrained = None

    embedding_size = embedding_pretrained.size(1) if embedding_pretrained is not None else 300
        
    kenel_num = 256
    kenel_size = [2,3,4]
    lr = 0.001
    epochs = 10
    batch_size = 128
    dropout = 0.5
    with open('word2id.json', 'r', encoding='utf-8') as f:
        word2id = json.load(f)
    vocab_size = len(word2id)