#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys import pandas as pd import numpy as np import sklearn from sklearn.model_selection import train_test_split from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn import preprocessing import imbalanced_ensemble as imbens import matplotlib.pyplot as plt import seaborn as sns from IPython.core.pylabtools import figsize subs = [] def container_count(str): ct = 0 for ch in str: if ch == ',': ct = ct + 1 return ct + 1 def pre_process(data): data.drop(columns=['id', 'entry_id', 'g_no', 'g_name', 'note_s', 'bill_no' , 'loginname', 'username', 'trade_name', 'agent_name', 'owner_name'], inplace=True) # 尝试g_model彻底拆分，没效果 # global subs # temp_data = pd.DataFrame(np.zeros((data.shape[0], len(subs))), columns=subs, index=range(0, data.shape[0])) # # for i in data.index: # string = data.loc[i, 'g_model'] # if isinstance(string, str): # elms = string.split('|') # for elm in elms: # if elm in subs: # temp_data.loc[i, elm] = 1 # print(temp_data['杂色,粗洗']) # print(data['g_model']) for field in ['hs', 'g_model', 'trade_curr', 'origin_country', 'g_unit', 'wrap_type' , 'dest', 'country', 'i_e_flag', 'trade_mode', 'traf_name', 'traf_mode']: data[field] = pd.Categorical(data[field]).codes status_dict = ['低', '中', '超高'] for field in ['decl_price', 'decl_total', 'g_qty', 'gross_wt', 'pack_no', 'net_wt']: data[field] = data[field].apply(lambda x: status_dict.index(x)) for field in ['trade_co_id', 'agent_co_id', 'owner_co_id']: data[field] = [int(x.split('_')[-1]) for x in data[field]] data['container_nums'] = data['container_nums'].apply(container_count) data['i_e_date'] = data['i_e_date'].apply(lambda x: 2 if np.isnan(x) else x) # data = pd.concat([data, temp_data], axis=1) return data def plot_cost_matrix(cost_matrix, title:str, **kwargs): ax = sns.heatmap(data=cost_matrix, **kwargs) ax.set_ylabel("Predicted Label") ax.set_xlabel("Ground Truth") ax.set_title(title) def imbens_classify(X_train, y_train, X_valid, y_valid, X_test): # estimator = DecisionTreeClassifier() # estimator = RandomForestClassifier() # estimator = SVC(probability=True) my_matrixs = [ [ [1, 20], [0.051, 1] ], [ [2, 20], [1, 2] ], [ [2, 10], [5, 1] ], [ [2, 100], [0.5, 2] ], [ [1, 0.05], [20, 1] ], ] plot_cost_matrix(my_matrixs[0], "test", annot=True, cmap='YlOrRd', vmax=6) plt.show() depth_best = 1 matrix_best = 0 n_best = 1 f1_best = 0 # estimator = DecisionTreeClassifier() # for var in range(1, 70): # clf = imbens.ensemble.SelfPacedEnsembleClassifier( # random_state=42, # base_estimator=estimator, # n_estimators=var # ).fit(X_train, y_train, # eval_datasets={ # 'valid': (X_valid, y_valid), # add validation data # }, # eval_metrics={ # 'f1': (sklearn.metrics.f1_score, {}), # 'f1_macro': (sklearn.metrics.f1_score, {'average': 'macro'}), # }, # # train_verbose={ # # 'granularity': 1, # # } # ) # # f1 = sklearn.metrics.f1_score(y_test, clf.predict(X_test)) # if f1 > f1_best: # f1_best = f1 # var_best = var # print(var_best, f1_best) # exit(-1) # # for depth in range(11, 100, 10): # estimator = DecisionTreeClassifier(max_depth=depth_best) # clf = imbens.ensemble.AsymBoostClassifier( # random_state=42, # base_estimator=estimator, # # n_estimators=50 # ).fit(X_train, y_train, # eval_datasets={ # 'valid': (X_valid, y_valid), # add validation data # }, # eval_metrics={ # 'f1_micro': (sklearn.metrics.f1_score, {'average': 'micro'}), # 'f1_macro': (sklearn.metrics.f1_score, {'average': 'macro'}), # }, # cost_matrix=my_matrixs[0] # ) # # f1_macro = sklearn.metrics.f1_score(y_test, clf.predict(X_test), # average='macro') # if f1_macro > f1_best: # f1_best = f1_macro # depth_best = depth # print("sub best depth: " + str(depth_best)) # # f1_best = 0 # for depth in range(depth_best - 10, depth_best + 10): # for matrix in range(0, len(my_matrixs)): # for n in range(1, 102): # estimator = DecisionTreeClassifier(max_depth=depth) # clf = imbens.ensemble.AsymBoostClassifier( # random_state=42, # base_estimator=estimator, # n_estimators=n # ).fit(X_train, y_train, # eval_datasets={ # 'valid': (X_valid, y_valid), # add validation data # }, # eval_metrics={ # 'f1_micro': ( # sklearn.metrics.f1_score, {'average': 'micro'}), # 'f1_macro': ( # sklearn.metrics.f1_score, {'average': 'macro'}), # }, # cost_matrix=my_matrixs[matrix], # # train_verbose={ # # 'granularity': 1, # # } # ) # # f1_macro = sklearn.metrics.f1_score(y_test, clf.predict(X_test), # average='macro') # if f1_macro > f1_best: # f1_best = f1_macro # depth_best = depth # matrix_best = matrix # n_best = n # print("max_depth: %d, matrix_num: %d, n_estimators: %d" # % (depth_best, matrix_best, n_best)) estimator = DecisionTreeClassifier(max_depth=34) clf = imbens.ensemble.AsymBoostClassifier( random_state=42, base_estimator=estimator, n_estimators=50 ).fit(X_train, y_train, eval_datasets={ 'valid': (X_valid, y_valid), # add validation data }, eval_metrics={ 'recall': (sklearn.metrics.f1_score, {}), 'f1_macro': (sklearn.metrics.f1_score, {'average': 'macro'}), }, cost_matrix=my_matrixs[1], train_verbose={ 'granularity': 1, } ) evaluate_predict(y_train, clf.predict(X_train), "train") evaluate_predict(y_valid, clf.predict(X_valid), "valid") fitted_ensembles = {'test': clf} visualizer = imbens.visualizer.ImbalancedEnsembleVisualizer( eval_datasets={ 'training': (X_train, y_train), 'validation': (X_valid, y_valid), }, eval_metrics={ 'f1_micro': (sklearn.metrics.f1_score, {'average': 'micro'}), 'f1_macro': (sklearn.metrics.f1_score, {'average': 'macro'}), 'recall': (sklearn.metrics.recall_score, {'average': 'binary'}) }, ) visualizer.fit(fitted_ensembles) fig, axes = visualizer.performance_lineplot() plt.show() return clf.predict(X_test) def evaluate_predict(y, y_pred, title=""): matrix = sklearn.metrics.confusion_matrix(y, y_pred) sns.heatmap(matrix, annot=True, cmap='Purples', fmt='g') plt.ylabel(u'True Value') plt.xlabel(u'Predict Value') plt.title(title) plt.show() print(title + "-----------------------------------------------------------") f1_micro = sklearn.metrics.f1_score(y, y_pred, average='micro') f1_macro = sklearn.metrics.f1_score(y, y_pred, average='macro') recall_1 = sklearn.metrics.recall_score(y, y_pred, average='binary') precision_1 = sklearn.metrics.precision_score(y, y_pred, average='binary') f1_1 = sklearn.metrics.f1_score(y, y_pred, average='binary') print('f1_micro: {:.3f}'.format(f1_micro)) print('f1_macro: {:.3f}'.format(f1_macro)) print('recall(passed=1): {:.3f}'.format(recall_1)) print('precision(passed=1): {:.3f}'.format(precision_1)) print('f1(passed=1): {:.3f}'.format(f1_1)) def main(argv): data = pd.read_csv('./train.csv', encoding='gbk') X_test = pd.read_csv('./test.csv', encoding='gbk') print(data.shape) # for string in data['g_model']: # if isinstance(string, str): # elementes = string.split('|') # for element in elementes: # if element not in subs: # subs.append(element) data = pre_process(data) X_test = pre_process(X_test) train, valid = train_test_split(data, test_size=0.2 , stratify=data[['passed']], random_state=0) X_train = train.drop(columns=['passed']) y_train = train['passed'] X_valid = valid.drop(columns=['passed']) y_valid = valid['passed'] print(type(X_train)) # plt.rcParams['figure.dpi'] = 300 colnm = X_train.columns.tolist() mcorr = X_train[colnm].corr( method="spearman") # 相关系数矩阵，即给出了任意两个变量之间的相关系数 mask = np.zeros_like(mcorr, dtype=np.bool) # 构造与mcorr同维数矩阵 为bool型 mask[np.triu_indices_from(mask)] = True # 角分线右侧为True cmap = sns.diverging_palette(220, 10, as_cmap=True) # 返回matplotlib colormap对象 g = sns.heatmap(mcorr, mask=mask, cmap=cmap, square=True, fmt='0.2f', xticklabels=False, yticklabels=False) # 热力图（看两两相似度） plt.show() X_train = preprocessing.scale(X_train) X_valid = preprocessing.scale(X_valid) X_test = preprocessing.scale(X_test) y_test_pred = imbens_classify(X_train, y_train, X_valid, y_valid, X_test) pd.DataFrame(y_test_pred).to_csv('./result.txt', index=False, header=False) if __name__ == '__main__': main(sys.argv)