|
|
|
|
|
import numpy as np
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
from pylab import mpl
|
|
|
|
|
|
|
|
|
|
|
|
mpl.rcParams["font.sans-serif"] = ["SimHei"]
|
|
|
|
|
|
mpl.rcParams["axes.unicode_minus"] = False
|
|
|
|
|
|
|
|
|
|
|
|
df_baidu = pd.read_csv('../data/baidu_index.csv',encoding = 'utf-8')# 百度流感指数
|
|
|
|
|
|
df_beijing = pd.read_csv('../data/beijin_zhoubao.csv',encoding = 'utf-8')# 北京传染病周报
|
|
|
|
|
|
df_liugan = pd.read_csv('../data/liugan_zhoubao.csv',encoding = 'utf-8')# 流感周报
|
|
|
|
|
|
df_hx = pd.read_csv('../data/hx_jijin_data.csv',encoding = 'utf-8')# 流感基金——华商医药医疗行业
|
|
|
|
|
|
df_gf = pd.read_csv('../data/gf_jijin_data.csv',encoding = 'utf-8')# 流感基金——广发创新医疗两年持有混合
|
|
|
|
|
|
# 确保日期列是日期类型
|
|
|
|
|
|
df_baidu['date'] = pd.to_datetime(df_baidu['date'])
|
|
|
|
|
|
df_beijing['date'] = pd.to_datetime(df_beijing['date'])
|
|
|
|
|
|
df_liugan['date'] = pd.to_datetime(df_liugan['date'])
|
|
|
|
|
|
df_hx['date'] = pd.to_datetime(df_hx['date'])
|
|
|
|
|
|
df_gf['date'] = pd.to_datetime(df_gf['date'])
|
|
|
|
|
|
df1 = df_baidu
|
|
|
|
|
|
df2 = df_beijing
|
|
|
|
|
|
df3 = df_liugan
|
|
|
|
|
|
df4 = df_hx
|
|
|
|
|
|
df5 = df_gf
|
|
|
|
|
|
# 创建一个完整的日期范围
|
|
|
|
|
|
all_dates = pd.date_range(start=min(df1['date'].min(), df2['date'].min(), df3['date'].min()),
|
|
|
|
|
|
end=max(df1['date'].max(), df2['date'].max(), df3['date'].max()))
|
|
|
|
|
|
# 重新索引每个DataFrame以包括所有日期
|
|
|
|
|
|
df1 = df1.set_index('date').reindex(all_dates).ffill().reset_index().rename(columns={'index': 'date'})
|
|
|
|
|
|
df2 = df2.set_index('date').reindex(all_dates).ffill().reset_index().rename(columns={'index': 'date'})
|
|
|
|
|
|
df3 = df3.set_index('date').reindex(all_dates).ffill().reset_index().rename(columns={'index': 'date'})
|
|
|
|
|
|
df4 = df4.set_index('date').reindex(all_dates).ffill().reset_index().rename(columns={'index': 'date'})
|
|
|
|
|
|
df5 = df5.set_index('date').reindex(all_dates).ffill().reset_index().rename(columns={'index': 'date'})
|
|
|
|
|
|
df1.drop(columns=['Unnamed: 0'], inplace=True)
|
|
|
|
|
|
df2.drop(columns=['Unnamed: 0'], inplace=True)
|
|
|
|
|
|
df3.drop(columns=['Unnamed: 0'], inplace=True)
|
|
|
|
|
|
df4.drop(columns=['Unnamed: 0'], inplace=True)
|
|
|
|
|
|
df5.drop(columns=['Unnamed: 0'], inplace=True)
|
|
|
|
|
|
# 合并数据集
|
|
|
|
|
|
df_merged = df1.merge(df2, on='date', how='outer').merge(df3, on='date', how='outer').merge(df4, on='date', how='outer').merge(df5, on='date', how='outer')
|
|
|
|
|
|
df_merged = df_merged[['date', 'liugan_index', 'beijing_number', 'infection_number','hx_jijin_data','gf_jijin_data']]
|
|
|
|
|
|
|
|
|
|
|
|
# 输出合并后的DataFrame
|
|
|
|
|
|
# print(df_merged.head(20))
|
|
|
|
|
|
#缺失值处理
|
|
|
|
|
|
# df = df_merged.dropna(how= 'any')
|
|
|
|
|
|
# 确保'date'列是日期格式，并设置为索引
|
|
|
|
|
|
df_merged['date'] = pd.to_datetime(df_merged['date'])
|
|
|
|
|
|
df_merged.set_index('date', inplace=True)
|
|
|
|
|
|
|
|
|
|
|
|
# 只对非日期列转换数据类型
|
|
|
|
|
|
numerical_columns = df_merged.columns.difference(['date']) # 排除'date'列
|
|
|
|
|
|
df_merged[numerical_columns] = df_merged[numerical_columns].astype(float)
|
|
|
|
|
|
|
|
|
|
|
|
# 确保数据类型正确，并查找是否有NaN或inf值
|
|
|
|
|
|
df_merged = df_merged.astype(float)
|
|
|
|
|
|
print("Initial NaN or Inf check:", df_merged.isin([np.inf, -np.inf]).sum(), df_merged.isna().sum())
|
|
|
|
|
|
|
|
|
|
|
|
# 处理NaN值和无穷大值
|
|
|
|
|
|
df_merged.replace([np.inf, -np.inf], np.nan, inplace=True)
|
|
|
|
|
|
df_merged.ffill() # 使用前向填充处理NaN值
|
|
|
|
|
|
df_merged.dropna(inplace=True) # 如果有剩余的NaN值，删除这些行
|
|
|
|
|
|
df_merged.to_csv('../data/merged_data.csv',encoding='utf-8')
|
|
|
|
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
|
|
|
|
scaler = MinMaxScaler()
|
|
|
|
|
|
# 选择需要归一化的列
|
|
|
|
|
|
columns_to_scale = ['liugan_index', 'beijing_number', 'infection_number','hx_jijin_data','gf_jijin_data']
|
|
|
|
|
|
# 对选定的列进行归一化处理
|
|
|
|
|
|
df_merged[columns_to_scale] = scaler.fit_transform(df_merged[columns_to_scale])
|
|
|
|
|
|
# 查看归一化后的数据
|
|
|
|
|
|
print(df_merged.head())
|
|
|
|
|
|
df_merged.to_csv('../data/merged_data.csv',index=True,encoding = 'utf-8')
|
