liurui/PriceForecast
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

市场数据库获取数据

Browse Source
This commit is contained in:
workpc 2024-12-26 15:58:14 +08:00
parent 0c13c27173
commit 48bea095a0
4 changed files with 38 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config_jingbo.py
View File

@ -219,7 +219,7 @@ query_data_list_item_nos_data = {
"data": {
"dateStart":"20200101",
"dateEnd":"20241231",
"dataItemNoList":["Brentzdj","Brentzgj"]
"dataItemNoList":["Brentzdj","Brentzgj"] # 数据项编码,代表 brent最低价和最高价
}
}

28
lib/dataread.py
View File

@ -139,8 +139,9 @@ def get_head_auth_report():
返回:
str: 如果登录成功，返回认证令牌；否则返回 None。
"""
logger.info("获取token中...")
# 发送 POST 请求到登录 URL，携带登录数据
login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 5))
login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 30))
# 将响应内容转换为 JSON 格式
text = json.loads(login_res.text)
@ -1419,10 +1420,7 @@ class EtaReader():
df1 = df1[df1['date'] <= datetime.datetime.now().strftime('%Y-%m-%d')]
logger.info(df1.head())
# logger.info(f'{df1.head()}')
# 保存到xlsx文件的sheet表
with pd.ExcelWriter(os.path.join(dataset,data_set)) as file:
df1.to_excel(file, sheet_name='指标数据', index=False)
df.to_excel(file, sheet_name='指标列表', index=False)
df_zhibiaoshuju = df1.copy()
df_zhibiaoliebiao = df.copy()
return df_zhibiaoshuju,df_zhibiaoliebiao
@ -1649,21 +1647,25 @@ def get_market_data(end_time,df):
query_data_list_item_nos_data['data']['dateEnd'] = end_time.replace('-','')
# 发送请求
headers = {"Authorization": token}
logger.info('获取数据中...')
items_res = requests.post(url=query_data_list_item_nos_url, headers=headers, json=query_data_list_item_nos_data, timeout=(3, 35))
json_data = json.loads(items_res.text)
df = pd.DataFrame(json_data['data'])
df3 = pd.DataFrame(json_data['data'])
# 按照dataItemNo 分组 得到多个dataframe ，最后根据dataDate merge 成一个dataframe
df2 = pd.DataFrame()
for i in df['dataItemNo'].unique():
df1 = df[df['dataItemNo'] == i]
for i in df3['dataItemNo'].unique():
df1 = df3[df3['dataItemNo'] == i]
df1 = df1[['dataDate', 'dataValue']]
df1 = df1.rename(columns={'dataValue': i})
df2 = pd.concat([df2, df1], axis=0)
df2 = df2.rename(columns={'dataDate': 'ds'})
if len(df2) == 0:
df2 = df1
continue
df2 = pd.merge(df2, df1, how='left')
df2 = df2.rename(columns={'dataDate': 'date'})
# 20240101 转换为 2024-01-01
df2['ds'] = pd.to_datetime(df2['ds'], format='%Y%m%d')
df2['ds'] = df2['ds'].dt.strftime('%Y-%m-%d')
df = pd.merge(df, df2, how='outer',on='ds')
df2['date'] = pd.to_datetime(df2['date'], format='%Y%m%d')
df2['date'] = df2['date'].dt.strftime('%Y-%m-%d')
df = pd.merge(df, df2, how='left',on='date')
return df
# 时间特征，年，月，一年的多少天，周几，第几周，第几季度，每月的第几天， 每季度的第几天，是否每月的第一天，是否每月的最后一天，是否每季度的第一天，是否每季度的最后一天，是否每年的第一天，是否每年的最后一天

11
main_yuanyou.py
View File

@ -76,7 +76,16 @@ def predict_main():
if is_market:
logger.info('从市场信息平台获取数据...')
df_zhibiaoshuju = get_market_data(end_time,df_zhibiaoshuju)
try:
df_zhibiaoshuju = get_market_data(end_time,df_zhibiaoshuju)
except :
logger.info('从市场信息平台获取数据失败')
# 保存到xlsx文件的sheet表
with pd.ExcelWriter(os.path.join(dataset,data_set)) as file:
df_zhibiaoshuju.to_excel(file, sheet_name='指标数据', index=False)
df_zhibiaoliebiao.to_excel(file, sheet_name='指标列表', index=False)
# 数据处理
df = datachuli(df_zhibiaoshuju, df_zhibiaoliebiao, y=y, dataset=dataset, add_kdj=add_kdj, is_timefurture=is_timefurture,

24
测试环境获取市场信息平台数据项.ipynb
View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 36,
"execution_count": 69,
"id": "31c0e11d-c87a-4e95-92a0-d1d09625e255",
"metadata": {},
"outputs": [],
@ -15,7 +15,7 @@
},
{
"cell_type": "code",
"execution_count": 37,
"execution_count": 70,
"id": "83c81b9e",
"metadata": {},
"outputs": [
@ -25,7 +25,7 @@
"'http://192.168.100.53:8080/jingbo-dev/api/server/login'"
]
},
"execution_count": 37,
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
@ -44,7 +44,7 @@
},
{
"cell_type": "code",
"execution_count": 38,
"execution_count": 71,
"id": "2b330ee3-c006-4ab1-8558-59c51ac8d86f",
"metadata": {},
"outputs": [
@ -59,7 +59,7 @@
" 'funcOperation': '获取token'}"
]
},
"execution_count": 38,
"execution_count": 71,
"metadata": {},
"output_type": "execute_result"
}
@ -70,7 +70,7 @@
},
{
"cell_type": "code",
"execution_count": 39,
"execution_count": 72,
"id": "dcb6100a-ed2b-4077-a1a9-361c6cb565f9",
"metadata": {},
"outputs": [],
@ -87,7 +87,7 @@
},
{
"cell_type": "code",
"execution_count": 40,
"execution_count": 73,
"id": "22c0c7c4",
"metadata": {},
"outputs": [
@ -95,7 +95,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzUyMjYxOTMsImp0aSI6IjdjNzllYTc2NTY0MTQxODdiOTg1ZTJlODVjMDIyZGE2In0.IOhOgFiEHKA1VdQx4dYubLmZ2ZwpHppm4XbJkpZXIJg', 'md5Token': 'b939d2aed088dbff085dac1863f1eea4'}, 'status': True}\n"
"{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzUyMzQ4MjAsImp0aSI6IjU5MWE4YWUzZGM1MjQxMWQ5Nzg5NGM5NjE1Mjc0MDZiIn0.qD81hhr7L4It2hM4K6uZlUs3iDRwebRG-FYSFWwYtkE', 'md5Token': '3b2ddd87d60b651a666ced63260655c3'}, 'status': True}\n"
]
}
],
@ -105,7 +105,7 @@
},
{
"cell_type": "code",
"execution_count": 41,
"execution_count": 74,
"id": "12077ead",
"metadata": {},
"outputs": [],
@ -115,7 +115,7 @@
},
{
"cell_type": "code",
"execution_count": 42,
"execution_count": 75,
"id": "a7ae21d1",
"metadata": {},
"outputs": [],
@ -149,7 +149,7 @@
},
{
"cell_type": "code",
"execution_count": 58,
"execution_count": 76,
"id": "54942e1a",
"metadata": {},
"outputs": [
@ -179,7 +179,7 @@
},
{
"cell_type": "code",
"execution_count": 59,
"execution_count": 77,
"id": "5272c84b",
"metadata": {},
"outputs": [