diff --git a/config_jingbo.py b/config_jingbo.py index 7374b09..ca3a5ac 100644 --- a/config_jingbo.py +++ b/config_jingbo.py @@ -219,7 +219,7 @@ query_data_list_item_nos_data = { "data": { "dateStart":"20200101", "dateEnd":"20241231", - "dataItemNoList":["Brentzdj","Brentzgj"] + "dataItemNoList":["Brentzdj","Brentzgj"] # 数据项编码,代表 brent最低价和最高价 } } diff --git a/lib/dataread.py b/lib/dataread.py index 596c548..21d9a96 100644 --- a/lib/dataread.py +++ b/lib/dataread.py @@ -139,8 +139,9 @@ def get_head_auth_report(): 返回: str: 如果登录成功，返回认证令牌；否则返回 None。 """ + logger.info("获取token中...") # 发送 POST 请求到登录 URL，携带登录数据 - login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 5)) + login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 30)) # 将响应内容转换为 JSON 格式 text = json.loads(login_res.text) @@ -1419,10 +1420,7 @@ class EtaReader(): df1 = df1[df1['date'] <= datetime.datetime.now().strftime('%Y-%m-%d')] logger.info(df1.head()) # logger.info(f'{df1.head()}') - # 保存到xlsx文件的sheet表 - with pd.ExcelWriter(os.path.join(dataset,data_set)) as file: - df1.to_excel(file, sheet_name='指标数据', index=False) - df.to_excel(file, sheet_name='指标列表', index=False) + df_zhibiaoshuju = df1.copy() df_zhibiaoliebiao = df.copy() return df_zhibiaoshuju,df_zhibiaoliebiao @@ -1649,21 +1647,25 @@ def get_market_data(end_time,df): query_data_list_item_nos_data['data']['dateEnd'] = end_time.replace('-','') # 发送请求 headers = {"Authorization": token} + logger.info('获取数据中...') items_res = requests.post(url=query_data_list_item_nos_url, headers=headers, json=query_data_list_item_nos_data, timeout=(3, 35)) json_data = json.loads(items_res.text) - df = pd.DataFrame(json_data['data']) + df3 = pd.DataFrame(json_data['data']) # 按照dataItemNo 分组 得到多个dataframe ，最后根据dataDate merge 成一个dataframe df2 = pd.DataFrame() - for i in df['dataItemNo'].unique(): - df1 = df[df['dataItemNo'] == i] + for i in df3['dataItemNo'].unique(): + df1 = df3[df3['dataItemNo'] == i] df1 = df1[['dataDate', 'dataValue']] df1 = df1.rename(columns={'dataValue': i}) - df2 = pd.concat([df2, df1], axis=0) - df2 = df2.rename(columns={'dataDate': 'ds'}) + if len(df2) == 0: + df2 = df1 + continue + df2 = pd.merge(df2, df1, how='left') + df2 = df2.rename(columns={'dataDate': 'date'}) # 20240101 转换为 2024-01-01 - df2['ds'] = pd.to_datetime(df2['ds'], format='%Y%m%d') - df2['ds'] = df2['ds'].dt.strftime('%Y-%m-%d') - df = pd.merge(df, df2, how='outer',on='ds') + df2['date'] = pd.to_datetime(df2['date'], format='%Y%m%d') + df2['date'] = df2['date'].dt.strftime('%Y-%m-%d') + df = pd.merge(df, df2, how='left',on='date') return df # 时间特征，年，月，一年的多少天，周几，第几周，第几季度，每月的第几天， 每季度的第几天，是否每月的第一天，是否每月的最后一天，是否每季度的第一天，是否每季度的最后一天，是否每年的第一天，是否每年的最后一天 diff --git a/main_yuanyou.py b/main_yuanyou.py index 83d3a48..5a4f63b 100644 --- a/main_yuanyou.py +++ b/main_yuanyou.py @@ -76,7 +76,16 @@ def predict_main(): if is_market: logger.info('从市场信息平台获取数据...') - df_zhibiaoshuju = get_market_data(end_time,df_zhibiaoshuju) + try: + df_zhibiaoshuju = get_market_data(end_time,df_zhibiaoshuju) + except : + logger.info('从市场信息平台获取数据失败') + + # 保存到xlsx文件的sheet表 + with pd.ExcelWriter(os.path.join(dataset,data_set)) as file: + df_zhibiaoshuju.to_excel(file, sheet_name='指标数据', index=False) + df_zhibiaoliebiao.to_excel(file, sheet_name='指标列表', index=False) + # 数据处理 df = datachuli(df_zhibiaoshuju, df_zhibiaoliebiao, y=y, dataset=dataset, add_kdj=add_kdj, is_timefurture=is_timefurture, diff --git a/测试环境获取市场信息平台数据项.ipynb b/测试环境获取市场信息平台数据项.ipynb index c3d2afd..248b05f 100644 --- a/测试环境获取市场信息平台数据项.ipynb +++ b/测试环境获取市场信息平台数据项.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 36, + "execution_count": 69, "id": "31c0e11d-c87a-4e95-92a0-d1d09625e255", "metadata": {}, "outputs": [], @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 70, "id": "83c81b9e", "metadata": {}, "outputs": [ @@ -25,7 +25,7 @@ "'http://192.168.100.53:8080/jingbo-dev/api/server/login'" ] }, - "execution_count": 37, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -44,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 71, "id": "2b330ee3-c006-4ab1-8558-59c51ac8d86f", "metadata": {}, "outputs": [ @@ -59,7 +59,7 @@ " 'funcOperation': '获取token'}" ] }, - "execution_count": 38, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 72, "id": "dcb6100a-ed2b-4077-a1a9-361c6cb565f9", "metadata": {}, "outputs": [], @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 73, "id": "22c0c7c4", "metadata": {}, "outputs": [ @@ -95,7 +95,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzUyMjYxOTMsImp0aSI6IjdjNzllYTc2NTY0MTQxODdiOTg1ZTJlODVjMDIyZGE2In0.IOhOgFiEHKA1VdQx4dYubLmZ2ZwpHppm4XbJkpZXIJg', 'md5Token': 'b939d2aed088dbff085dac1863f1eea4'}, 'status': True}\n" + "{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzUyMzQ4MjAsImp0aSI6IjU5MWE4YWUzZGM1MjQxMWQ5Nzg5NGM5NjE1Mjc0MDZiIn0.qD81hhr7L4It2hM4K6uZlUs3iDRwebRG-FYSFWwYtkE', 'md5Token': '3b2ddd87d60b651a666ced63260655c3'}, 'status': True}\n" ] } ], @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 74, "id": "12077ead", "metadata": {}, "outputs": [], @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 75, "id": "a7ae21d1", "metadata": {}, "outputs": [], @@ -149,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 76, "id": "54942e1a", "metadata": {}, "outputs": [ @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 77, "id": "5272c84b", "metadata": {}, "outputs": [