特征停更预警消息推送
This commit is contained in:
workpc
parent
dabff4b157
commit
c893350b7d
120
config_jingbo.py
120
config_jingbo.py
@ -92,63 +92,14 @@ ClassifyId = 1214
|
||||
|
||||
### 报告上传配置
|
||||
# 变量定义--线上环境
|
||||
login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login"
|
||||
upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave"
|
||||
upload_warning_url = "http://10.200.32.39/jingbo-api/api/basicBuiness/crudeOilWarning/save"
|
||||
|
||||
login_data = {
|
||||
"data": {
|
||||
"account": "api_dev",
|
||||
"password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=",
|
||||
"tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
|
||||
"terminal": "API"
|
||||
},
|
||||
"funcModule": "API",
|
||||
"funcOperation": "获取token"
|
||||
}
|
||||
|
||||
|
||||
|
||||
upload_data = {
|
||||
"funcModule":'研究报告信息',
|
||||
"funcOperation":'上传原油价格预测报告',
|
||||
"data":{
|
||||
"ownerAccount":'27663', #报告所属用户账号 27663 - 刘小朋
|
||||
"reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
|
||||
"fileName": '', #文件名称
|
||||
"fileBase64": '' ,#文件内容base64
|
||||
"categoryNo":'yyjgycbg', # 研究报告分类编码
|
||||
"smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
|
||||
"reportEmployeeCode":"E40482" ,# 报告人 E40482 - 管理员 0000027663 - 刘小朋
|
||||
"reportDeptCode" :"002000621000", # 报告部门 - 002000621000 SH期货研究部
|
||||
"productGroupCode":"RAW_MATERIAL" # 商品分类
|
||||
}
|
||||
}
|
||||
|
||||
warning_data = {
|
||||
"funcModule":'原油特征停更预警',
|
||||
"funcOperation":'原油特征停更预警',
|
||||
"data":{
|
||||
'WARNING_TYPE_NAME':'特征数据停更预警',
|
||||
'WARNING_CONTENT':'',
|
||||
'WARNING_DATE':''
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
# # 变量定义--测试环境
|
||||
# login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
|
||||
# upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
|
||||
# # upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
|
||||
# upload_warning_url = "http://192.168.100.53:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
|
||||
|
||||
# login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login"
|
||||
# upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave"
|
||||
# upload_warning_url = "http://10.200.32.39/jingbo-api/api/basicBuiness/crudeOilWarning/save"
|
||||
|
||||
# login_data = {
|
||||
# "data": {
|
||||
# "account": "api_test",
|
||||
# # "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=", # Shihua@123456
|
||||
# "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", # 123456
|
||||
# "account": "api_dev",
|
||||
# "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=",
|
||||
# "tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
|
||||
# "terminal": "API"
|
||||
# },
|
||||
@ -156,18 +107,20 @@ warning_data = {
|
||||
# "funcOperation": "获取token"
|
||||
# }
|
||||
|
||||
|
||||
|
||||
# upload_data = {
|
||||
# "funcModule":'研究报告信息',
|
||||
# "funcOperation":'上传原油价格预测报告',
|
||||
# "data":{
|
||||
# "ownerAccount":'arui', #报告所属用户账号
|
||||
# "ownerAccount":'27663', #报告所属用户账号 27663 - 刘小朋
|
||||
# "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
|
||||
# "fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称
|
||||
# "fileName": '', #文件名称
|
||||
# "fileBase64": '' ,#文件内容base64
|
||||
# "categoryNo":'yyjgycbg', # 研究报告分类编码
|
||||
# "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
|
||||
# "reportEmployeeCode":"E40116", # 报告人
|
||||
# "reportDeptCode" :"D0044" ,# 报告部门
|
||||
# "reportEmployeeCode":"E40482" ,# 报告人 E40482 - 管理员 0000027663 - 刘小朋
|
||||
# "reportDeptCode" :"002000621000", # 报告部门 - 002000621000 SH期货研究部
|
||||
# "productGroupCode":"RAW_MATERIAL" # 商品分类
|
||||
# }
|
||||
# }
|
||||
@ -182,6 +135,53 @@ warning_data = {
|
||||
# }
|
||||
# }
|
||||
|
||||
|
||||
|
||||
# # 变量定义--测试环境
|
||||
login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
|
||||
upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
|
||||
# upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
|
||||
upload_warning_url = "http://192.168.100.53:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
|
||||
|
||||
|
||||
login_data = {
|
||||
"data": {
|
||||
"account": "api_test",
|
||||
# "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=", # Shihua@123456
|
||||
"password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", # 123456
|
||||
"tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
|
||||
"terminal": "API"
|
||||
},
|
||||
"funcModule": "API",
|
||||
"funcOperation": "获取token"
|
||||
}
|
||||
|
||||
upload_data = {
|
||||
"funcModule":'研究报告信息',
|
||||
"funcOperation":'上传原油价格预测报告',
|
||||
"data":{
|
||||
"ownerAccount":'arui', #报告所属用户账号
|
||||
"reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
|
||||
"fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称
|
||||
"fileBase64": '' ,#文件内容base64
|
||||
"categoryNo":'yyjgycbg', # 研究报告分类编码
|
||||
"smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
|
||||
"reportEmployeeCode":"E40116", # 报告人
|
||||
"reportDeptCode" :"D0044" ,# 报告部门
|
||||
"productGroupCode":"RAW_MATERIAL" # 商品分类
|
||||
}
|
||||
}
|
||||
|
||||
warning_data = {
|
||||
"funcModule":'原油特征停更预警',
|
||||
"funcOperation":'原油特征停更预警',
|
||||
"data":{
|
||||
'WARNING_TYPE_NAME':'特征数据停更预警',
|
||||
'WARNING_CONTENT':'',
|
||||
'WARNING_DATE':''
|
||||
}
|
||||
}
|
||||
|
||||
### 线上开关
|
||||
# is_train = True # 是否训练
|
||||
# is_debug = False # 是否调试
|
||||
@ -198,14 +198,14 @@ warning_data = {
|
||||
### 开关
|
||||
is_train = True # 是否训练
|
||||
is_debug = False # 是否调试
|
||||
is_eta = True # 是否使用eta接口
|
||||
is_eta = False # 是否使用eta接口
|
||||
is_timefurture = True # 是否使用时间特征
|
||||
is_fivemodels = False # 是否使用之前保存的最佳的5个模型
|
||||
is_edbcode = True # 特征使用edbcoding列表中的
|
||||
is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
|
||||
is_update_eta = False # 预测结果上传到eta
|
||||
is_update_report = False # 是否上传报告
|
||||
is_update_warning_data = False # 是否上传预警数据
|
||||
is_update_warning_data = True # 是否上传预警数据
|
||||
|
||||
### 北京环境数据库jbsh_test
|
||||
# url: jdbc:mysql://192.168.101.27:3306/jingbo_test?autoReconnect=true&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&useSSL=false&serverTimezone=GMT%2B8&nullCatalogMeansCurrent=true
|
||||
|
||||
@ -134,25 +134,38 @@ def upload_warning_data(warning_data):
|
||||
return None
|
||||
|
||||
|
||||
def upload_warning_info(last_update_times_df,y_last_update_time):
|
||||
# def upload_warning_info(last_update_times_df,y_last_update_time):
|
||||
# logger.info(f'上传预警信息')
|
||||
# try:
|
||||
# warning_data_df = last_update_times_df[last_update_times_df['warning_date']<y_last_update_time][['stop_update_period','warning_date','last_update_time','update_period','feature']]
|
||||
# warning_data_df.columns = ['停更周期','预警日期','最后更新时间','更新周期','特征名称']
|
||||
# if len(warning_data_df) > 0:
|
||||
# content = '原油特征指标预警信息：\n\n'
|
||||
# warning_data_df = warning_data_df.sort_values(by='停更周期',ascending=False)
|
||||
# fixed_length = 20
|
||||
# warning_data_df['特征名称'] = warning_data_df['特征名称'].str.replace(" ", "")
|
||||
# content = warning_data_df.to_string(index=False, col_space=fixed_length)
|
||||
|
||||
# else:
|
||||
# logger.info(f'没有需要上传的预警信息')
|
||||
# content = '没有需要维护的特征指标'
|
||||
# warning_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
# warning_data['data']['WARNING_DATE'] = warning_date
|
||||
# warning_data['data']['WARNING_CONTENT'] = content
|
||||
|
||||
# upload_warning_data(warning_data)
|
||||
# logger.info(f'上传预警信息成功')
|
||||
# except Exception as e:
|
||||
# logger.error(f'上传预警信息失败：{e}')
|
||||
|
||||
|
||||
def upload_warning_info(df_count):
|
||||
logger.info(f'上传预警信息')
|
||||
try:
|
||||
warning_data_df = last_update_times_df[last_update_times_df['warning_date']<y_last_update_time][['stop_update_period','warning_date','last_update_time','update_period','feature']]
|
||||
warning_data_df.columns = ['停更周期','预警日期','最后更新时间','更新周期','特征名称']
|
||||
if len(warning_data_df) > 0:
|
||||
content = '原油特征指标预警信息：\n\n'
|
||||
warning_data_df = warning_data_df.sort_values(by='停更周期',ascending=False)
|
||||
fixed_length = 20
|
||||
warning_data_df['特征名称'] = warning_data_df['特征名称'].str.replace(" ", "")
|
||||
content = warning_data_df.to_string(index=False, col_space=fixed_length)
|
||||
|
||||
else:
|
||||
logger.info(f'没有需要上传的预警信息')
|
||||
content = '没有需要维护的特征指标'
|
||||
warning_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
warning_date = datetime.datetime.now().strftime('%Y-%m-%d')
|
||||
content = f'{warning_date}有{df_count}个停更'
|
||||
warning_data['data']['WARNING_DATE'] = warning_date
|
||||
warning_data['data']['WARNING_CONTENT'] = content
|
||||
|
||||
upload_warning_data(warning_data)
|
||||
logger.info(f'上传预警信息成功')
|
||||
except Exception as e:
|
||||
@ -571,13 +584,13 @@ def datachuli(df_zhibiaoshuju,df_zhibiaoliebiao,datecol='date',end_time='',y='y'
|
||||
# 获取start_year年到end_time的数据
|
||||
df = df[df['ds'].dt.year >= start_year]
|
||||
df = df[df['ds'] <= end_time]
|
||||
last_update_times_df,y_last_update_time = create_feature_last_update_time(df)
|
||||
logger.info(f'删除预警的特征前数据量：{df.shape}')
|
||||
columns_to_drop = last_update_times_df[last_update_times_df['warning_date'] < y_last_update_time ]['feature'].values.tolist()
|
||||
df = df.drop(columns = columns_to_drop)
|
||||
logger.info(f'删除预警的特征后数据量：{df.shape}')
|
||||
if is_update_warning_data:
|
||||
upload_warning_info(last_update_times_df,y_last_update_time)
|
||||
# last_update_times_df,y_last_update_time = create_feature_last_update_time(df)
|
||||
# logger.info(f'删除预警的特征前数据量：{df.shape}')
|
||||
# columns_to_drop = last_update_times_df[last_update_times_df['warning_date'] < y_last_update_time ]['feature'].values.tolist()
|
||||
# df = df.drop(columns = columns_to_drop)
|
||||
# logger.info(f'删除预警的特征后数据量：{df.shape}')
|
||||
# if is_update_warning_data:
|
||||
# upload_warning_info(last_update_times_df,y_last_update_time)
|
||||
# 去掉近最后数据对应的日期在六月以前的列，删除近2月的数据是常熟的列
|
||||
current_date = datetime.datetime.now()
|
||||
two_months_ago = current_date - timedelta(days=180)
|
||||
|
||||
@ -139,8 +139,9 @@ def predict_main():
|
||||
sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',))
|
||||
|
||||
try:
|
||||
if is_weekday:
|
||||
logger.info('今天是周一，更新预测模型')
|
||||
# if is_weekday:
|
||||
if True:
|
||||
logger.info('今天是周一，发送特征预警')
|
||||
# 上传预警信息到数据库
|
||||
warning_data_df = df_zhibiaoliebiao.copy()
|
||||
warning_data_df = warning_data_df[warning_data_df['停更周期']> 3 ][['指标名称', '指标id', '频度','更新周期','指标来源','最后更新时间','停更周期']]
|
||||
@ -153,11 +154,13 @@ def predict_main():
|
||||
# 插入数据之前查询表数据然后新增id列
|
||||
existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine)
|
||||
if not existing_data.empty:
|
||||
max_id = existing_data['id'].max()
|
||||
warning_data_df['id'] = range(max_id + 1, max_id + 1 + len(warning_data_df))
|
||||
max_id = existing_data['ID'].astype(int).max()
|
||||
warning_data_df['ID'] = range(max_id + 1, max_id + 1 + len(warning_data_df))
|
||||
else:
|
||||
warning_data_df['id'] = range(1, 1 + len(warning_data_df))
|
||||
warning_data_df['ID'] = range(1, 1 + len(warning_data_df))
|
||||
warning_data_df.to_sql(table_name, con=engine, if_exists='append', index=False)
|
||||
if is_update_warning_data:
|
||||
upload_warning_info(len(warning_data_df))
|
||||
except:
|
||||
logger.info('上传预警信息到数据库失败')
|
||||
|
||||
@ -170,25 +173,25 @@ def predict_main():
|
||||
row, col = df.shape
|
||||
|
||||
now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
|
||||
ex_Model(df,
|
||||
horizon=horizon,
|
||||
input_size=input_size,
|
||||
train_steps=train_steps,
|
||||
val_check_steps=val_check_steps,
|
||||
early_stop_patience_steps=early_stop_patience_steps,
|
||||
is_debug=is_debug,
|
||||
dataset=dataset,
|
||||
is_train=is_train,
|
||||
is_fivemodels=is_fivemodels,
|
||||
val_size=val_size,
|
||||
test_size=test_size,
|
||||
settings=settings,
|
||||
now=now,
|
||||
etadata=etadata,
|
||||
modelsindex=modelsindex,
|
||||
data=data,
|
||||
is_eta=is_eta,
|
||||
)
|
||||
# ex_Model(df,
|
||||
# horizon=horizon,
|
||||
# input_size=input_size,
|
||||
# train_steps=train_steps,
|
||||
# val_check_steps=val_check_steps,
|
||||
# early_stop_patience_steps=early_stop_patience_steps,
|
||||
# is_debug=is_debug,
|
||||
# dataset=dataset,
|
||||
# is_train=is_train,
|
||||
# is_fivemodels=is_fivemodels,
|
||||
# val_size=val_size,
|
||||
# test_size=test_size,
|
||||
# settings=settings,
|
||||
# now=now,
|
||||
# etadata=etadata,
|
||||
# modelsindex=modelsindex,
|
||||
# data=data,
|
||||
# is_eta=is_eta,
|
||||
# )
|
||||
|
||||
|
||||
logger.info('模型训练完成')
|
||||
|
||||
116
测试环境登录接口调试.ipynb
116
测试环境登录接口调试.ipynb
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user