diff --git a/config_jingbo.py b/config_jingbo.py index 83cd9b9..8c514fa 100644 --- a/config_jingbo.py +++ b/config_jingbo.py @@ -89,8 +89,7 @@ data = { ClassifyId = 1214 - -################################################################################################################ 变量定义--线上环境 +# 变量定义--线上环境 # server_host = '10.200.32.39' # login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login" # upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave" @@ -111,7 +110,6 @@ ClassifyId = 1214 # } - # upload_data = { # "funcModule":'研究报告信息', # "funcOperation":'上传原油价格预测报告', @@ -119,12 +117,12 @@ ClassifyId = 1214 # "groupNo":'', # 用户组id # "ownerAccount":'27663', #报告所属用户账号 27663 - 刘小朋 # "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST -# "fileName": '', #文件名称 +# "fileName": '', #文件名称 # "fileBase64": '' ,#文件内容base64 # "categoryNo":'yyjgycbg', # 研究报告分类编码 # "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码 -# "reportEmployeeCode":"E40482" ,# 报告人 E40482 - 管理员 0000027663 - 刘小朋 -# "reportDeptCode" :"002000621000", # 报告部门 - 002000621000 SH期货研究部 +# "reportEmployeeCode":"E40482" ,# 报告人 E40482 - 管理员 0000027663 - 刘小朋 +# "reportDeptCode" :"002000621000", # 报告部门 - 002000621000 SH期货研究部 # "productGroupCode":"RAW_MATERIAL" # 商品分类 # } # } @@ -151,7 +149,6 @@ ClassifyId = 1214 # } - # push_data_value_list_data = { # "funcModule": "数据表信息列表", # "funcOperation": "新增", @@ -189,13 +186,12 @@ ClassifyId = 1214 # # 生产环境数据库 # host = 'rm-2zehj3r1n60ttz9x5.mysql.rds.aliyuncs.com' # port = 3306 -# dbusername ='jingbo' +# dbusername ='jingbo' # password = 'shihua@123' # dbname = 'jingbo' # table_name = 'v_tbl_crude_oil_warning' - # # 变量定义--测试环境 server_host = '192.168.100.53' # 内网 # server_host = '183.242.74.28' # 外网 @@ -208,6 +204,8 @@ upload_warning_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/cru query_data_list_item_nos_url = f"http://{server_host}:8080/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos" # 上传数据项值 push_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/dw/dataValue/pushDataValueList" +# 上传停更数据到市场信息平台 +push_waring_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/crudeSaveOrupdate" login_data = { "data": { @@ -281,6 +279,26 @@ push_data_value_list_data = { } ] } + +push_waring_data_value_list_data = { + "data": { + "crudeOilWarningDtoList": [ + { + "lastUpdateDate": "20240501", + "updateSuspensionCycle": 1, + "dataSource": "8", + "frequency": "1", + "indicatorName": "美元指数", + "indicatorId": "myzs001", + "warningDate": "2024-05-13" + } + ], + "dataSource": "8" + }, + "funcModule": "商品数据同步", + "funcOperation": "同步" +} + # 八大维度数据项编码 bdwd_items = { 'ciri': 'yyycbdwdcr', diff --git a/lib/dataread.py b/lib/dataread.py index 769569b..fcec16a 100644 --- a/lib/dataread.py +++ b/lib/dataread.py @@ -40,15 +40,6 @@ pdfmetrics.registerFont(TTFont('SimSun', 'SimSun.ttf')) plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签 plt.rcParams['axes.unicode_minus'] = False # 用来正常显示负号 -# from config_jingbo_pro import * -# from config_jingbo import * -# from config_jingbo_yuedu import * -# from config_yongan import * -# from config_juxiting import * -# from config_juxiting_zhoudu import * -# from config_juxiting_pro import * - -# from config_jingbo import logger global_config = { # 核心配置项 @@ -93,6 +84,9 @@ global_config = { # 上传数据项数据 'push_data_value_list_url': None, 'push_data_value_list_data': None, + # 上传停更预警数据 + 'push_waring_data_value_list_url': None, + 'push_waring_data_value_list_data': None, # 字段映射 'offsite_col': None, # 站点字段 @@ -1222,6 +1216,14 @@ class Config: def push_data_value_list_data( self): return global_config['push_data_value_list_data'] + @property + def push_waring_data_value_list_url( + self): return global_config['push_waring_data_value_list_url'] + + @property + def push_waring_data_value_list_data( + self): return global_config['push_waring_data_value_list_data'] + @property def bdwd_items(self): return global_config['bdwd_items'] @@ -2203,6 +2205,43 @@ def push_market_data(data): return json_data +def push_waring_market_data(data): + ''' + 上传停更预警数据到市场信息平台 + data: 预测价格数据,示例： + { + "data": { + "crudeOilWarningDtoList": [ + { + "lastUpdateDate": "20240510", + "updateSuspensionCycle": 1, + "dataSource": "8", + "frequency": "1", + "indicatorName": "美元指数", + "indicatorId": "myzs001", + "warningDate": "2024-05-12" + } + ], + "dataSource": "8" + }, + "funcModule": "商品数据同步", + "funcOperation": "同步" +} + ''' + # 获取token + token = get_head_auth_report() + # 定义请求参数 + global_config['push_waring_data_value_list_data']['data']["crudeOilWarningDtoList"] = data + # 发送请求 + headers = {"Authorization": token} + config.logger.info('上传数据中...') + items_res = requests.post(url=config.push_waring_data_value_list_url, headers=headers, + json=config.push_waring_data_value_list_data, timeout=(3, 35)) + json_data = json.loads(items_res.text) + config.logger.info(f"上传结果:{json_data}") + return json_data + + def get_high_low_data(df): # 读取excel 从第五行开始 df1 = pd.read_excel(os.path.join(config.dataset, '数据项下载.xls'), header=5, names=[ diff --git a/main_yuanyou.py b/main_yuanyou.py index dc30ac2..74fb1c6 100644 --- a/main_yuanyou.py +++ b/main_yuanyou.py @@ -63,6 +63,10 @@ global_config.update({ 'push_data_value_list_url': push_data_value_list_url, 'push_data_value_list_data': push_data_value_list_data, + # 上传数据项 + 'push_waring_data_value_list_url': push_waring_data_value_list_url, + 'push_waring_data_value_list_data': push_waring_data_value_list_data, + # eta 配置 'APPID': APPID, 'SECRET': SECRET, @@ -83,7 +87,6 @@ global_config.update({ }) - def push_market_value(): config.logger.info('发送预测结果到市场信息平台') # 读取预测数据和模型评估数据 @@ -120,13 +123,13 @@ def push_market_value(): predictdata = [ { "dataItemNo": global_config['bdwd_items']['ciri'], - "dataDate": global_config['end_time'].replace('-',''), + "dataDate": global_config['end_time'].replace('-', ''), "dataStatus": "add", "dataValue": first_mean }, { "dataItemNo": global_config['bdwd_items']['benzhou'], - "dataDate": global_config['end_time'].replace('-',''), + "dataDate": global_config['end_time'].replace('-', ''), "dataStatus": "add", "dataValue": last_mean } @@ -141,8 +144,6 @@ def push_market_value(): config.logger.error(f"推送数据失败: {e}") - - def predict_main(): """ 主预测函数，用于从 ETA 获取数据、处理数据、训练模型并进行预测。 @@ -197,7 +198,7 @@ def predict_main(): # 获取数据 if is_eta: logger.info('从eta获取数据...') - + df_zhibiaoshuju, df_zhibiaoliebiao = etadata.get_eta_api_yuanyou_data( data_set=data_set, dataset=dataset) # 原始数据，未处理 @@ -338,94 +339,122 @@ def predict_main(): sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',)) - try: - if is_weekday: - # if True: - logger.info('今天是周一，发送特征预警') - # 上传预警信息到数据库 - warning_data_df = df_zhibiaoliebiao.copy() - warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[ - '指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']] - # 重命名列名 - warning_data_df = warning_data_df.rename(columns={'指标名称': 'INDICATOR_NAME', '指标id': 'INDICATOR_ID', '频度': 'FREQUENCY', - '更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'UPDATE_SUSPENSION_CYCLE'}) - from sqlalchemy import create_engine - import urllib - global password - if '@' in password: - password = urllib.parse.quote_plus(password) + # try: + # # if is_weekday: + # if True: + # logger.info('今天是周一，发送特征预警') + # # 上传预警信息到数据库 + # warning_data_df = df_zhibiaoliebiao.copy() + # warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[ + # '指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']] + # # 重命名列名 + # warning_data_df = warning_data_df.rename(columns={'指标名称': 'INDICATOR_NAME', '指标id': 'INDICATOR_ID', '频度': 'FREQUENCY', + # '更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'UPDATE_SUSPENSION_CYCLE'}) + # from sqlalchemy import create_engine + # import urllib + # global password + # if '@' in password: + # password = urllib.parse.quote_plus(password) - engine = create_engine( - f'mysql+pymysql://{dbusername}:{password}@{host}:{port}/{dbname}') - warning_data_df['WARNING_DATE'] = datetime.date.today().strftime( - "%Y-%m-%d %H:%M:%S") - warning_data_df['TENANT_CODE'] = 'T0004' - # 插入数据之前查询表数据然后新增id列 - existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine) - if not existing_data.empty: - max_id = existing_data['ID'].astype(int).max() - warning_data_df['ID'] = range( - max_id + 1, max_id + 1 + len(warning_data_df)) - else: - warning_data_df['ID'] = range(1, 1 + len(warning_data_df)) - warning_data_df.to_sql( - table_name, con=engine, if_exists='append', index=False) - if is_update_warning_data: - upload_warning_info(len(warning_data_df)) - except: - logger.info('上传预警信息到数据库失败') + # engine = create_engine( + # f'mysql+pymysql://{dbusername}:{password}@{host}:{port}/{dbname}') + # warning_data_df['WARNING_DATE'] = datetime.date.today().strftime( + # "%Y-%m-%d %H:%M:%S") + # warning_data_df['TENANT_CODE'] = 'T0004' + # # 插入数据之前查询表数据然后新增id列 + # existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine) + # if not existing_data.empty: + # max_id = existing_data['ID'].astype(int).max() + # warning_data_df['ID'] = range( + # max_id + 1, max_id + 1 + len(warning_data_df)) + # else: + # warning_data_df['ID'] = range(1, 1 + len(warning_data_df)) + # warning_data_df.to_sql( + # table_name, con=engine, if_exists='append', index=False) + # if is_update_warning_data: + # upload_warning_info(len(warning_data_df)) + # except: + # logger.info('上传预警信息到数据库失败') - if is_corr: - df = corr_feature(df=df) + # try: + # if is_weekday: + if True: + logger.info('发送特征预警') + # 获取取消订阅的指标ID - df1 = df.copy() # 备份一下，后面特征筛选完之后加入ds y 列用 - logger.info(f"开始训练模型...") - row, col = df.shape + # 上传预警信息到数据库 + warning_data_df = df_zhibiaoliebiao.copy() + warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[ + '指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']] + # 重命名列名 + warning_data_df = warning_data_df.rename(columns={'指标名称': 'indicatorName', '指标id': 'indicatorId', '频度': 'frequency', + '更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'updateSuspensionCycle'}) - now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - ex_Model(df, - horizon=global_config['horizon'], - input_size=global_config['input_size'], - train_steps=global_config['train_steps'], - val_check_steps=global_config['val_check_steps'], - early_stop_patience_steps=global_config['early_stop_patience_steps'], - is_debug=global_config['is_debug'], - dataset=global_config['dataset'], - is_train=global_config['is_train'], - is_fivemodels=global_config['is_fivemodels'], - val_size=global_config['val_size'], - test_size=global_config['test_size'], - settings=global_config['settings'], - now=now, - etadata=etadata, - modelsindex=global_config['modelsindex'], - data=data, - is_eta=global_config['is_eta'], - end_time=global_config['end_time'], - ) + warning_data_df['warningDate'] = datetime.date.today().strftime( + "%Y-%m-%d %H:%M:%S") + warning_data_df['dataSource'] = 8 + data = warning_data_df.to_json(orient='records', force_ascii=False) + data = data.replace('日度', '1') + data = data.replace('周度', '2') + data = data.replace('月度', '3') + data = json.loads(data) + push_waring_market_data(data) + # if is_update_warning_data: + # upload_warning_info(len(warning_data_df)) + # except: + # logger.info('上传预警信息到数据库失败') - logger.info('模型训练完成') + # if is_corr: + # df = corr_feature(df=df) - logger.info('训练数据绘图ing') - model_results3 = model_losss(sqlitedb, end_time=end_time) - logger.info('训练数据绘图end') + # df1 = df.copy() # 备份一下，后面特征筛选完之后加入ds y 列用 + # logger.info(f"开始训练模型...") + # row, col = df.shape - # 模型报告 - logger.info('制作报告ing') - title = f'{settings}--{end_time}-预测报告' # 报告标题 - reportname = f'Brent原油大模型日度预测--{end_time}.pdf' # 报告文件名 - reportname = reportname.replace(':', '-') # 替换冒号 - brent_export_pdf(dataset=dataset, - num_models=5 if is_fivemodels else 22, time=end_time, - reportname=reportname, - inputsize = global_config['horizon'], - sqlitedb=sqlitedb - ), + # now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + # ex_Model(df, + # horizon=global_config['horizon'], + # input_size=global_config['input_size'], + # train_steps=global_config['train_steps'], + # val_check_steps=global_config['val_check_steps'], + # early_stop_patience_steps=global_config['early_stop_patience_steps'], + # is_debug=global_config['is_debug'], + # dataset=global_config['dataset'], + # is_train=global_config['is_train'], + # is_fivemodels=global_config['is_fivemodels'], + # val_size=global_config['val_size'], + # test_size=global_config['test_size'], + # settings=global_config['settings'], + # now=now, + # etadata=etadata, + # modelsindex=global_config['modelsindex'], + # data=data, + # is_eta=global_config['is_eta'], + # end_time=global_config['end_time'], + # ) - logger.info('制作报告end') - logger.info('模型训练完成') + # logger.info('模型训练完成') - push_market_value() + # logger.info('训练数据绘图ing') + # model_results3 = model_losss(sqlitedb, end_time=end_time) + # logger.info('训练数据绘图end') + + # # 模型报告 + # logger.info('制作报告ing') + # title = f'{settings}--{end_time}-预测报告' # 报告标题 + # reportname = f'Brent原油大模型日度预测--{end_time}.pdf' # 报告文件名 + # reportname = reportname.replace(':', '-') # 替换冒号 + # brent_export_pdf(dataset=dataset, + # num_models=5 if is_fivemodels else 22, time=end_time, + # reportname=reportname, + # inputsize=global_config['horizon'], + # sqlitedb=sqlitedb + # ), + + # logger.info('制作报告end') + # logger.info('模型训练完成') + + # push_market_value() # # LSTM 单变量模型 # ex_Lstm(df,input_seq_len=input_size,output_seq_len=horizon,is_debug=is_debug,dataset=dataset) diff --git a/report/xgboost_report.md b/report/xgboost_report.md deleted file mode 100644 index b49ee8c..0000000 --- a/report/xgboost_report.md +++ /dev/null @@ -1,40 +0,0 @@ -# XGBoost 价格预测分析报告 2025-05-08 - -## 一、模型实现 - -### 1. 特征工程 - -- 使用 5 期历史滞后特征 -- 预测未来 10 个时间步长 -- 数据集分割比例：80% 训练集 / 20% 测试集 - -### 2. 模型配置 - -| | learning_rate | max_depth | n_estimators | -| :----------- | ------------: | --------: | -----------: | -| 最佳参数组合 | 1 | 4 | 100 | - -## 二、性能评估 - -### 多步预测误差分析 - -| 预测步长 | 均方根误差(RMSE) | R² 分数 | -| -------: | ---------------: | -------: | -| 1 | 2.30044 | 0.828193 | -| 2 | 2.74815 | 0.756595 | -| 3 | 3.16346 | 0.679378 | -| 4 | 3.62202 | 0.58126 | -| 5 | 3.77657 | 0.546517 | - -## 三、预测结果 - -### 未来 5 日价格预测 - -![预测结果](价格预测.png) -| | 预测值 | -|:--------------------|---------:| -| 2025-03-14 00:00:00 | 68.8125 | -| 2025-03-15 00:00:00 | 68.6643 | -| 2025-03-16 00:00:00 | 69.283 | -| 2025-03-17 00:00:00 | 71.7288 | -| 2025-03-18 00:00:00 | 68.8356 | diff --git a/report/日度价格预测_最佳模型.pkl b/report/日度价格预测_最佳模型.pkl deleted file mode 100644 index 9867117..0000000 Binary files a/report/日度价格预测_最佳模型.pkl and /dev/null differ