From 830f369dc6b8383ce3240dec7fd9518bba53c6fe Mon Sep 17 00:00:00 2001 From: workpc Date: Mon, 7 Jul 2025 10:37:23 +0800 Subject: [PATCH] =?UTF-8?q?=E8=81=9A=E7=83=AF=E7=83=83=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config_juxiting.py | 254 ++++++++++++++++++++++++--------------------- main_juxiting.py | 112 ++++++++++++++++++-- 2 files changed, 237 insertions(+), 129 deletions(-) diff --git a/config_juxiting.py b/config_juxiting.py index 50c79d2..a15bcae 100644 --- a/config_juxiting.py +++ b/config_juxiting.py @@ -1,3 +1,4 @@ +from decimal import Decimal import logging import os import logging.handlers @@ -86,7 +87,6 @@ bdwdname = [ ] - modelsindex = [{ 'NHITS': 'SELF0000077', 'Informer': 'SELF0000078', @@ -137,140 +137,138 @@ data = { ClassifyId = 1161 - -# 变量定义--线上环境 -server_host = '10.200.32.39' -login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login" -upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave" -upload_warning_url = "http://10.200.32.39/jingbo-api/api/basicBuiness/crudeOilWarning/save" -query_data_list_item_nos_url = f"http://{server_host}/jingbo-api/api/warehouse/dwDataItem/queryDataListItemNos" -# 上传数据项值 -push_data_value_list_url = f"http://{server_host}/jingbo-api/api/dw/dataValue/pushDataValueList" -上传停更数据到市场信息平台 -push_waring_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/crudeSaveOrupdate" -获取预警数据中取消订阅指标ID -get_waring_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/dataList" +# # 变量定义--线上环境 +# server_host = '10.200.32.39' +# login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login" +# upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave" +# upload_warning_url = "http://10.200.32.39/jingbo-api/api/basicBuiness/crudeOilWarning/save" +# query_data_list_item_nos_url = f"http://{server_host}/jingbo-api/api/warehouse/dwDataItem/queryDataListItemNos" +# # 上传数据项值 +# push_data_value_list_url = f"http://{server_host}/jingbo-api/api/dw/dataValue/pushDataValueList" +# # 上传停更数据到市场信息平台 +# push_waring_data_value_list_url = f"http://{server_host}/jingbo-api/api/basicBuiness/crudeOilWarning/crudeSaveOrupdate" +# # 获取预警数据中取消订阅指标ID +# get_waring_data_value_list_url = f"http://{server_host}/jingbo-api/api/basicBuiness/crudeOilWarning/dataList" -login_data = { - "data": { - "account": "api_dev", - "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", - "tenantHashCode": "8a4577dbd919675758d57999a1e891fe", - "terminal": "API" - }, - "funcModule": "API", - "funcOperation": "获取token" -} +# login_data = { +# "data": { +# "account": "api_dev", +# "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", +# "tenantHashCode": "8a4577dbd919675758d57999a1e891fe", +# "terminal": "API" +# }, +# "funcModule": "API", +# "funcOperation": "获取token" +# } -upload_data = { - "funcModule":'研究报告信息', - "funcOperation":'上传聚烯烃PP价格预测报告', - "data":{ -    "groupNo":'000211' # 用户组编号 -        "ownerAccount":'36541', #报告所属用户账号  36541 - 贾青雪 -        "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST -        "fileName": '', #文件名称 -        "fileBase64": '' ,#文件内容base64 -        "categoryNo":'jxtjgycbg', # 研究报告分类编码 -        "smartBusinessClassCode":'JXTJGYCBG', #分析报告分类编码 -        "reportEmployeeCode":"E40482" ,# 报告人  E40482  - 管理员  0000027663 - 刘小朋   -        "reportDeptCode" :"JXTJGYCBG", # 报告部门 - 002000621000  SH期货研究部   -        "productGroupCode":"RAW_MATERIAL"  # 商品分类 -  } -} +# upload_data = { +# "funcModule": '研究报告信息', +# "funcOperation": '上传聚烯烃PP价格预测报告', +# "data": { +# "groupNo": '000211' # 用户组编号 +# "ownerAccount": '36541', # 报告所属用户账号  36541 - 贾青雪 +# "reportType": 'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST +# "fileName": '', # 文件名称 +# "fileBase64": '', # 文件内容base64 +# "categoryNo": 'jxtjgycbg', # 研究报告分类编码 +# "smartBusinessClassCode": 'JXTJGYCBG', # 分析报告分类编码 +# "reportEmployeeCode": "E40482", # 报告人  E40482  - 管理员  0000027663 - 刘小朋   +# "reportDeptCode": "JXTJGYCBG", # 报告部门 - 002000621000  SH期货研究部   +# "productGroupCode": "RAW_MATERIAL"   # 商品分类 +# } +# } -warning_data = { - "funcModule": '原油特征停更预警', - "funcOperation": '原油特征停更预警', - "data": { - "groupNo": "000211", - 'WARNING_TYPE_NAME': '特征数据停更预警', - 'WARNING_CONTENT': '', - 'WARNING_DATE': '' - } -} +# warning_data = { +# "funcModule": '原油特征停更预警', +# "funcOperation": '原油特征停更预警', +# "data": { +# "groupNo": "000211", +# 'WARNING_TYPE_NAME': '特征数据停更预警', +# 'WARNING_CONTENT': '', +# 'WARNING_DATE': '' +# } +# } -query_data_list_item_nos_data = { - "funcModule": "数据项", - "funcOperation": "查询", - "data": { - "dateStart":"20200101", - "dateEnd":"20241231", - "dataItemNoList":["Brentzdj","Brentzgj"] # 数据项编码,代表 brent最低价和最高价 - } -} +# query_data_list_item_nos_data = { +# "funcModule": "数据项", +# "funcOperation": "查询", +# "data": { +# "dateStart": "20200101", +# "dateEnd": "20241231", +# "dataItemNoList": ["Brentzdj", "Brentzgj"] # 数据项编码,代表 brent最低价和最高价 +# } +# } -push_data_value_list_data = { - "funcModule": "数据表信息列表", - "funcOperation": "新增", - "data": [ - {"dataItemNo": "91230600716676129", - "dataDate": "20230113", - "dataStatus": "add", - "dataValue": 100.11 - }, - {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY", - "dataDate": "20230113", - "dataStatus": "add", - "dataValue": 100.55 - }, - {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY", - "dataDate": "20230113", - "dataStatus": "add", - "dataValue": 100.55 - } - ] -} +# push_data_value_list_data = { +# "funcModule": "数据表信息列表", +# "funcOperation": "新增", +# "data": [ +# {"dataItemNo": "91230600716676129", +# "dataDate": "20230113", +# "dataStatus": "add", +# "dataValue": 100.11 +# }, +# {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY", +# "dataDate": "20230113", +# "dataStatus": "add", +# "dataValue": 100.55 +# }, +# {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY", +# "dataDate": "20230113", +# "dataStatus": "add", +# "dataValue": 100.55 +# } +# ] +# } -push_waring_data_value_list_data = { - "data": { - "crudeOilWarningDtoList": [ - { - "lastUpdateDate": "20240501", - "updateSuspensionCycle": 1, - "dataSource": "9", - "frequency": "1", - "indicatorName": "美元指数", - "indicatorId": "myzs001", - "warningDate": "2024-05-13" - } - ], - "dataSource": "9" - }, - "funcModule": "商品数据同步", - "funcOperation": "同步" -} +# push_waring_data_value_list_data = { +# "data": { +# "crudeOilWarningDtoList": [ +# { +# "lastUpdateDate": "20240501", +# "updateSuspensionCycle": 1, +# "dataSource": "9", +# "frequency": "1", +# "indicatorName": "美元指数", +# "indicatorId": "myzs001", +# "warningDate": "2024-05-13" +# } +# ], +# "dataSource": "9" +# }, +# "funcModule": "商品数据同步", +# "funcOperation": "同步" +# } -get_waring_data_value_list_data = { - "data": "9", "funcModule": "商品数据同步", "funcOperation": "同步"} +# get_waring_data_value_list_data = { +# "data": "9", "funcModule": "商品数据同步", "funcOperation": "同步"} -# 八大维度数据项编码 -bdwd_items = { - 'ciri': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE', - 'benzhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE01', - 'cizhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE02', - 'gezhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE03', - 'ciyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE04', - 'cieryue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE05', - 'cisanyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE06', - 'cisiyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE07', -} +# # 八大维度数据项编码 +# bdwd_items = { +# 'ciri': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE', +# 'benzhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE01', +# 'cizhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE02', +# 'gezhou': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE03', +# 'ciyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE04', +# 'cieryue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE05', +# 'cisanyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE06', +# 'cisiyue': '91371600MAC3TYFN6M|LSBM00007|FORECAST_PRICE07', +# } -# 生产环境数据库 -host = 'rm-2zehj3r1n60ttz9x5.mysql.rds.aliyuncs.com' -port = 3306 -dbusername ='jingbo' -password = 'shihua@123' -dbname = 'jingbo' -table_name = 'v_tbl_crude_oil_warning' - +# # 生产环境数据库 +# host = 'rm-2zehj3r1n60ttz9x5.mysql.rds.aliyuncs.com' +# port = 3306 +# dbusername = 'jingbo' +# password = 'shihua@123' +# dbname = 'jingbo' +# table_name = 'v_tbl_crude_oil_warning' # 变量定义--测试环境 @@ -408,6 +406,20 @@ password = '123456' dbname = 'jingbo_test' table_name = 'v_tbl_crude_oil_warning' +DEFAULT_CONFIG = { + 'feature_factor_frequency': 'D', + 'strategy_id': 2, + 'model_evaluation_id': 1, + 'tenant_code': '', + 'version_num': Decimal(1), + 'delete_flag': '0', + 'create_user': 'admin', + 'create_date': datetime.datetime.now(), + 'update_user': 'admin', + 'update_date': datetime.datetime.now(), + 'oil_code': 'PP', + 'oil_name': 'PP期货', +} # 开关 is_train = True # 是否训练 @@ -419,10 +431,12 @@ is_fivemodels = False # 是否使用之前保存的最佳的5个模型 is_edbcode = False # 特征使用edbcoding列表中的 is_edbnamelist = False # 自定义特征，对应上面的edbnamelist is_update_eta = True # 预测结果上传到eta -is_update_report = True # 是否上传报告 +is_update_report = False # 是否上传报告 is_update_warning_data = True # 是否上传预警数据 +is_update_predict_value = True # 是否上传预测值到市场信息平台 is_del_corr = 0.6 # 是否删除相关性高的特征,取值为 0-1 ，0 为不删除，0.6 表示删除相关性小于0.6的特征 is_del_tow_month = True # 是否删除两个月不更新的特征 +is_bdwd = False # 是否使用八大维度 # 连接到数据库 diff --git a/main_juxiting.py b/main_juxiting.py index 19d3f6a..f42f6f3 100644 --- a/main_juxiting.py +++ b/main_juxiting.py @@ -2,7 +2,7 @@ from lib.dataread import * from config_juxiting import * -from lib.tools import SendMail, exception_logger +from lib.tools import SendMail, exception_logger, convert_df_to_pydantic, exception_logger, get_modelsname from models.nerulforcastmodels import ex_Model_Juxiting, model_losss_juxiting, pp_export_pdf import datetime import torch @@ -23,6 +23,8 @@ global_config.update({ 'is_update_report': is_update_report, 'settings': settings, + 'bdwdname': bdwdname, + # 模型参数 'data_set': data_set, @@ -35,19 +37,18 @@ global_config.update({ 'modelsindex': modelsindex, 'rote': rote, 'bdwd_items': bdwd_items, - 'bdwdname':bdwdname, - # 特征工程开关 'is_del_corr': is_del_corr, 'is_del_tow_month': is_del_tow_month, 'is_eta': is_eta, 'is_update_eta': is_update_eta, 'is_fivemodels': is_fivemodels, + 'is_update_predict_value': is_update_predict_value, 'early_stop_patience_steps': early_stop_patience_steps, # 时间参数 'start_year': start_year, - 'end_time': end_time, + 'end_time': end_time or datetime.datetime.now().strftime("%Y-%m-%d"), 'freq': freq, # 保持列表结构 # 接口配置 @@ -90,6 +91,10 @@ global_config.update({ # 数据库配置 'sqlitedb': sqlitedb, + 'bdwd_items': bdwd_items, + 'is_bdwd': is_bdwd, + 'db_mysql': db_mysql, + 'DEFAULT_CONFIG': DEFAULT_CONFIG, }) @@ -150,6 +155,91 @@ def push_market_value(): config.logger.error(f"推送数据失败: {e}") +def sql_inset_predict(global_config): + df = pd.read_csv(os.path.join(config.dataset, 'predict.csv')) + df['created_dt'] = pd.to_datetime(df['created_dt']) + df['ds'] = pd.to_datetime(df['ds']) + # 获取次日预测结果 + next_day_df = df[df['ds'] == df['ds'].min()] + # 获取本周预测结果 + this_week_df = df[df['ds'] == df['ds'].max()] + + wd = ['day_price', 'week_price'] + model_name_list, model_id_name_dict = get_modelsname(df, global_config) + + PRICE_COLUMNS = [ + 'day_price', 'week_price', 'second_week_price', 'next_week_price', + 'next_month_price', 'next_february_price', 'next_march_price', 'next_april_price' + ] + + params_list = [] + for df, price_type in zip([next_day_df, this_week_df], wd): + + update_columns = [ + "feature_factor_frequency = VALUES(feature_factor_frequency)", + "oil_code = VALUES(oil_code)", + "oil_name = VALUES(oil_name)", + "data_date = VALUES(data_date)", + "market_price = VALUES(market_price)", + f"{price_type} = VALUES({price_type})", + "model_evaluation_id = VALUES(model_evaluation_id)", + "tenant_code = VALUES(tenant_code)", + "version_num = VALUES(version_num)", + "delete_flag = VALUES(delete_flag)", + "update_user = VALUES(update_user)", + "update_date = VALUES(update_date)" + ] + + insert_query = f""" + INSERT INTO v_tbl_predict_prediction_results ( + feature_factor_frequency, strategy_id, oil_code, oil_name, data_date, + market_price, day_price, week_price, second_week_price, next_week_price, + next_month_price, next_february_price, next_march_price, next_april_price, + model_evaluation_id, model_id, tenant_code, version_num, delete_flag, + create_user, create_date, update_user, update_date + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + ON DUPLICATE KEY UPDATE + {', '.join(update_columns)} + """ + + next_day_df = df[['ds', 'created_dt'] + model_name_list] + + pydantic_results = convert_df_to_pydantic( + next_day_df, model_id_name_dict, global_config) + if pydantic_results: + + for result in pydantic_results: + price_values = [None] * len(PRICE_COLUMNS) + price_index = PRICE_COLUMNS.index(price_type) + price_values[price_index] = next_day_df[model_id_name_dict[result.model_id]].values[0] + + params = ( + result.feature_factor_frequency, + result.strategy_id, + global_config['DEFAULT_CONFIG']['oil_code'], + global_config['DEFAULT_CONFIG']['oil_name'], + next_day_df['created_dt'].values[0], + result.market_price, + *price_values, + result.model_evaluation_id, + result.model_id, + result.tenant_code, + 1, + '0', + result.create_user, + result.create_date, + result.update_user, + result.update_date + ) + params_list.append(params) + affected_rows = config.db_mysql.execute_batch_insert( + insert_query, params_list) + config.logger.info(f"成功插入或更新 {affected_rows} 条记录") + config.db_mysql.close() + + def predict_main(): """ 主预测函数，用于从 ETA 获取数据、处理数据、训练模型并进行预测。 @@ -358,13 +448,13 @@ def predict_main(): '指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']] # 重命名列名 warning_data_df = warning_data_df.rename(columns={'指标名称': 'indicatorName', '指标id': 'indicatorId', '频度': 'frequency', - '更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'updateSuspensionCycle'}) + '更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'updateSuspensionCycle'}) warning_data_df['warningDate'] = datetime.date.today().strftime( "%Y-%m-%d %H:%M:%S") warning_data_df['dataSource'] = 9 - - if len(quxiaodingyueidlist)>0: + + if len(quxiaodingyueidlist) > 0: # 去掉取消订阅的指标 print(warning_data_df.shape) warning_data_df = warning_data_df[~warning_data_df['indicatorId'].isin( @@ -376,7 +466,8 @@ def predict_main(): warning_data = warning_data.replace('周度', '2') warning_data = warning_data.replace('月度', '3') warning_data = json.loads(warning_data) - push_waring_market_data(warning_data,dataSource=warning_data_df['dataSource'].values[0]) + push_waring_market_data( + warning_data, dataSource=warning_data_df['dataSource'].values[0]) # if is_update_warning_data: # upload_warning_info(len(warning_data_df)) # except: @@ -430,6 +521,7 @@ def predict_main(): logger.info('模型训练完成') push_market_value() + sql_inset_predict(global_config) # # LSTM 单变量模型 # ex_Lstm(df,input_seq_len=input_size,output_seq_len=horizon,is_debug=is_debug,dataset=dataset) @@ -464,6 +556,8 @@ if __name__ == '__main__': # logger.info(f'预测失败：{e}') # continue - predict_main() + # predict_main() # push_market_value() + + sql_inset_predict(global_config)