|
|
|
|
|
# 数据预处理界面
|
|
|
|
|
|
import json
|
|
|
|
|
|
import os
|
|
|
|
|
|
|
|
|
|
|
|
import django
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
import psycopg2
|
|
|
|
|
|
import streamlit as st
|
|
|
|
|
|
import streamlit.components.v1 as components
|
|
|
|
|
|
|
|
|
|
|
|
os.chdir('D:/python/djangoProject/test_Bootstrap')
|
|
|
|
|
|
# 设置DJANGO_SETTINGS_MODULE环境变量
|
|
|
|
|
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_Bootstrap.settings')
|
|
|
|
|
|
# 初始化Django设置
|
|
|
|
|
|
django.setup()
|
|
|
|
|
|
|
|
|
|
|
|
# 定义数据库连接参数
|
|
|
|
|
|
DB_PARAMS = {
|
|
|
|
|
|
'host': 'localhost',
|
|
|
|
|
|
'port': '5432',
|
|
|
|
|
|
'dbname': 'liugan_yuce',
|
|
|
|
|
|
'user': 'postgres',
|
|
|
|
|
|
'password': '123456',
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# 定义目标数据表
|
|
|
|
|
|
TABLES = [
|
|
|
|
|
|
'app_test_baidudata', 'app_test_beijingweekdata', 'app_test_jijindata',
|
|
|
|
|
|
'app_test_liuganweekdata', 'app_test_stockdata'
|
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def connect_to_pg():
|
|
|
|
|
|
"""
|
|
|
|
|
|
连接到 PostgreSQL 数据库并获取指定数据表。
|
|
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
|
dfs (dict): 键为数据表名（去除 'app01_' 前缀），值为对应数据表的 DataFrame 对象。
|
|
|
|
|
|
"""
|
|
|
|
|
|
try:
|
|
|
|
|
|
with psycopg2.connect(**DB_PARAMS) as conn: # 用 with 语句确保数据库连接被正确关闭
|
|
|
|
|
|
dfs = {
|
|
|
|
|
|
table[9:]: pd.read_sql(f'select * from public.{table};', conn)
|
|
|
|
|
|
for table in TABLES
|
|
|
|
|
|
}
|
|
|
|
|
|
print('{:*^30}'.format('成功链接 PostgreSQL'))
|
|
|
|
|
|
except Exception as error:
|
|
|
|
|
|
print(f"发现错误：{error}")
|
|
|
|
|
|
exit()
|
|
|
|
|
|
# print(dfs)
|
|
|
|
|
|
return dfs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def merge_dfs(dfs):
|
|
|
|
|
|
"""
|
|
|
|
|
|
合并 DataFrame 对象。
|
|
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
|
dfs (dict): 键为数据表名，值为对应数据表的 DataFrame 对象。
|
|
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
|
df (DataFrame): 合并后的 DataFrame 对象。
|
|
|
|
|
|
"""
|
|
|
|
|
|
df_merged = pd.DataFrame({'date': []})
|
|
|
|
|
|
|
|
|
|
|
|
for df in dfs.values(): # 应该把时间序列补充完整，而不是有部分时间缺失
|
|
|
|
|
|
del df['id'] # 删除 'id' 列
|
|
|
|
|
|
df_merged = pd.merge(df_merged, df, how='outer', on='date')
|
|
|
|
|
|
# df_merged['date'] = pd.to_datetime(df_merged['date'])
|
|
|
|
|
|
df_merged.set_index('date', inplace=True)
|
|
|
|
|
|
print(df_merged)
|
|
|
|
|
|
print('开始保存数据')
|
|
|
|
|
|
df_merged.to_csv('row_data.csv',encoding='utf-8')
|
|
|
|
|
|
print('插值')
|
|
|
|
|
|
# 对缺失值进行线性插值(其他方法?多项插值?)
|
|
|
|
|
|
df_merged = df_merged.interpolate()
|
|
|
|
|
|
print(df_merged)
|
|
|
|
|
|
# 如果有剩余的NaN值，删除这些行
|
|
|
|
|
|
df_merged.dropna(inplace=True)
|
|
|
|
|
|
|
|
|
|
|
|
return df_merged
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def normalize_df(df):
|
|
|
|
|
|
"""
|
|
|
|
|
|
对 DataFrame 对象进行最小最大标准化。
|
|
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
|
df (DataFrame): 要进行标准化的 DataFrame 对象。
|
|
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
|
df_normalized (DataFrame): 进行最小最大标准化后的 DataFrame 对象。
|
|
|
|
|
|
"""
|
|
|
|
|
|
# 如果列的数据类型是布尔值、有符号整型、无符号整型、浮点数或复数浮点数的话，就进行最大最小标准化，否则保留原列的数据
|
|
|
|
|
|
df_normalized = df.apply(lambda x: (x - x.min()) / (x.max() - x.min())
|
|
|
|
|
|
if x.dtype.kind in 'biufc' else x)
|
|
|
|
|
|
|
|
|
|
|
|
return df_normalized
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run():
|
|
|
|
|
|
# 连接数据库，获取数据表数据
|
|
|
|
|
|
dfs = connect_to_pg()
|
|
|
|
|
|
|
|
|
|
|
|
# 将获取的数据进行合并
|
|
|
|
|
|
merged_df = merge_dfs(dfs)
|
|
|
|
|
|
|
|
|
|
|
|
# 将合并后的数据进行最小最大标准化
|
|
|
|
|
|
# normalized_df = normalize_df(merged_df)
|
|
|
|
|
|
print("展示最终数据")
|
|
|
|
|
|
print(merged_df)
|
|
|
|
|
|
# 将标准化后的数据保存为 CSV 文件
|
|
|
|
|
|
# normalized_df.to_csv('normalized_df.csv', index_label='date')
|
|
|
|
|
|
merged_df.to_csv('normalized_df.csv', index_label='date')
|
|
|
|
|
|
# 打印完成信息
|
|
|
|
|
|
print('{:*^30}'.format('PostgreSQL数据读取完成'))
|
|
|
|
|
|
return merged_df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
run()
|
|
|
|
|
|
|
|
|
|
|
|
#解释滚动文本框
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 解释性文本
|
|
|
|
|
|
explanation_text = """
|
|
|
|
|
|
由于用于模型训练的数据集包括多种类型的时间序列数据，其中一些数据是周数据，例如“国家流感中心周报数据”。这些数据集由于采集频率和时段的不同，常常存在长度不一致和缺失值问题。如果直接使用这些不完整的数据进行模型训练，可能会导致模型的性能下降，甚至无法正常工作。故对数据集应用线性插值方法填补缺失值。对于每个缺失点，使用其相邻的已知数据点进行线性插值。插值完成后将处理后的各个数据集进行整合，生成一个完整的时间序列数据集，为后续的模型训练做好准备。
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
# 展示文本框
|
|
|
|
|
|
st.subheader('预处理说明')
|
|
|
|
|
|
st.text_area("", explanation_text, height=200)
|
|
|
|
|
|
|
|
|
|
|
|
# 可视化预处理后数据
|
|
|
|
|
|
df = pd.read_csv('app_test/normalized_df.csv')
|
|
|
|
|
|
columns = df.columns.tolist()
|
|
|
|
|
|
list_dict = {}
|
|
|
|
|
|
for column in columns:
|
|
|
|
|
|
list_dict[column] = json.dumps(df[column].tolist())
|
|
|
|
|
|
# print(list_dict)
|
|
|
|
|
|
#处理后数据下载
|
|
|
|
|
|
#相关性数据下载
|
|
|
|
|
|
def convert_df_to_csv(df):
|
|
|
|
|
|
return df.to_csv(index=False).encode('utf-8')
|
|
|
|
|
|
csv = convert_df_to_csv(df)
|
|
|
|
|
|
st.download_button(
|
|
|
|
|
|
label="下载预处理后数据",
|
|
|
|
|
|
data=csv,
|
|
|
|
|
|
file_name=f"预处理后数据.csv",
|
|
|
|
|
|
mime='text/csv'
|
|
|
|
|
|
)
|
|
|
|
|
|
# HTML内容，包括嵌入 ECharts 的代码
|
|
|
|
|
|
|
|
|
|
|
|
html_content = f"""
|
|
|
|
|
|
<script src="https://cdn.bootcdn.net/ajax/libs/echarts/5.5.0/echarts.min.js"></script>
|
|
|
|
|
|
<div id="main" style="width:700px; height:300px;background:#ccffff"></div>
|
|
|
|
|
|
<script>
|
|
|
|
|
|
var main = echarts.init(document.getElementById("main"));
|
|
|
|
|
|
|
|
|
|
|
|
option = {{
|
|
|
|
|
|
backgroundColor: "#f5f5f5",
|
|
|
|
|
|
title: {{
|
|
|
|
|
|
text: '预处理后数据'
|
|
|
|
|
|
}},
|
|
|
|
|
|
tooltip: {{
|
|
|
|
|
|
trigger: 'axis'
|
|
|
|
|
|
}},
|
|
|
|
|
|
legend: {{
|
|
|
|
|
|
data: ['国家流感中心', '北京疾控中心', '基金', '股票', '百度指数']
|
|
|
|
|
|
}},
|
|
|
|
|
|
grid: {{
|
|
|
|
|
|
left: '3%',
|
|
|
|
|
|
right: '4%',
|
|
|
|
|
|
bottom: '3%',
|
|
|
|
|
|
containLabel: true
|
|
|
|
|
|
}},
|
|
|
|
|
|
toolbox: {{
|
|
|
|
|
|
feature: {{
|
|
|
|
|
|
saveAsImage: {{}}
|
|
|
|
|
|
}}
|
|
|
|
|
|
}},
|
|
|
|
|
|
xAxis: {{
|
|
|
|
|
|
type: 'category',
|
|
|
|
|
|
boundaryGap: false,
|
|
|
|
|
|
data: {list_dict['date']}
|
|
|
|
|
|
}},
|
|
|
|
|
|
yAxis: {{
|
|
|
|
|
|
type: 'value',
|
|
|
|
|
|
min: 0,
|
|
|
|
|
|
max: 1,
|
|
|
|
|
|
}},
|
|
|
|
|
|
series: [
|
|
|
|
|
|
{{
|
|
|
|
|
|
name: '国家流感中心',
|
|
|
|
|
|
type: 'line',
|
|
|
|
|
|
data: {list_dict['infection_number_x']}
|
|
|
|
|
|
}},
|
|
|
|
|
|
{{
|
|
|
|
|
|
name: '北京疾控中心',
|
|
|
|
|
|
type: 'line',
|
|
|
|
|
|
data: {list_dict['infection_number_y']}
|
|
|
|
|
|
}},
|
|
|
|
|
|
{{
|
|
|
|
|
|
name: '基金',
|
|
|
|
|
|
type: 'line',
|
|
|
|
|
|
data: {list_dict['jijin_data']}
|
|
|
|
|
|
}},
|
|
|
|
|
|
{{
|
|
|
|
|
|
name: '股票',
|
|
|
|
|
|
type: 'line',
|
|
|
|
|
|
data: {list_dict['shoupan']}
|
|
|
|
|
|
}},
|
|
|
|
|
|
{{
|
|
|
|
|
|
name: '百度指数',
|
|
|
|
|
|
type: 'line',
|
|
|
|
|
|
data: {list_dict['liugan_index']}
|
|
|
|
|
|
}}
|
|
|
|
|
|
]
|
|
|
|
|
|
}};
|
|
|
|
|
|
console.log(option);
|
|
|
|
|
|
main.setOption(option);
|
|
|
|
|
|
</script>
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
# 使用 Streamlit 的 HTML 函数将 HTML 内容嵌入页面中
|
|
|
|
|
|
components.html(html_content, height=350)
|
