添加原油特征停更预警

This commit is contained in:
workpc 2024-11-18 14:27:04 +08:00
parent da929edbac
commit 9b7f64535f
10 changed files with 34079 additions and 69746 deletions

View File

@ -121,38 +121,49 @@ upload_data = {
# # 变量定义--测试环境
# login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
# upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
# # upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
# upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
upload_warning_url = "http://192.168.100.53:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
# login_data = {
# "data": {
# "account": "api_test",
# "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=",
# "tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
# "terminal": "API"
# },
# "funcModule": "API",
# "funcOperation": "获取token"
# }
login_data = {
"data": {
"account": "api_test",
# "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=", # Shihua@123456
"password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", # 123456
"tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
"terminal": "API"
},
"funcModule": "API",
"funcOperation": "获取token"
}
# upload_data = {
# "funcModule":'研究报告信息',
# "funcOperation":'上传原油价格预测报告',
# "data":{
# "ownerAccount":'arui', #报告所属用户账号
# "reportType":'OIL_PRICE_FORECAST', # 报告类型固定为OIL_PRICE_FORECAST
# "fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称
# "fileBase64": '' ,#文件内容base64
# "categoryNo":'yyjgycbg', # 研究报告分类编码
# "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
# "reportEmployeeCode":"E40116", # 报告人
# "reportDeptCode" :"D0044" ,# 报告部门
# "productGroupCode":"RAW_MATERIAL" # 商品分类
# }
# }
upload_data = {
"funcModule":'研究报告信息',
"funcOperation":'上传原油价格预测报告',
"data":{
"ownerAccount":'arui', #报告所属用户账号
"reportType":'OIL_PRICE_FORECAST', # 报告类型固定为OIL_PRICE_FORECAST
"fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称
"fileBase64": '' ,#文件内容base64
"categoryNo":'yyjgycbg', # 研究报告分类编码
"smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
"reportEmployeeCode":"E40116", # 报告人
"reportDeptCode" :"D0044" ,# 报告部门
"productGroupCode":"RAW_MATERIAL" # 商品分类
}
}
warning_data = {
"funcModule":'原油特征停更预警',
"funcOperation":'原油特征停更预警',
"data":{
'WARNING_TYPE_NAME':'日度数据',
'WARNING_CONTENT':'',
'WARNING_DATE':''
}
}
### 线上开关
# is_train = True # 是否训练
@ -164,6 +175,8 @@ upload_data = {
# is_edbnamelist = False # 自定义特征对应上面的edbnamelist
# is_update_eta = True # 预测结果上传到eta
# is_update_report = True # 是否上传报告
# is_update_warning_data = False # 是否上传预警数据
### 开关
is_train = True # 是否训练
@ -175,7 +188,7 @@ is_edbcode = False # 特征使用edbcoding列表中的
is_edbnamelist = False # 自定义特征对应上面的edbnamelist
is_update_eta = False # 预测结果上传到eta
is_update_report = False # 是否上传报告
is_update_warning_data = True # 是否上传预警数据
# 数据截取日期
end_time = '' # 数据截取日期

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@ -114,7 +114,19 @@ def upload_report_data(token, upload_data):
logger.info("报告上传失败")
return None
def upload_warning_data(token, warning_data):
warning_data = warning_data
headers = {"Authorization": token}
logger.info("预警上传中...")
logger.info(f"token:{token}")
logger.info(f"warning_data:{warning_data}" )
upload_res = requests.post(url=upload_warning_url, headers=headers, json=warning_data, timeout=(3, 15))
if upload_res:
return upload_res
else:
logger.info("预警上传失败")
return None
# 统计特征频度
def featurePindu(dataset):
@ -498,12 +510,26 @@ def datachuli(df_zhibiaoshuju,df_zhibiaoliebiao,datecol='date',end_time='',y='y'
logger.info(f'删除预警的特征后数据量:{df.shape}')
logger.info(f'删除两月不更新特征前数据量:{df.shape}')
if is_update_warning_data:
logger.info(f'上传预警信息')
try:
warning_data_df = last_update_times_df[last_update_times_df['feature'].isin(columns_to_drop)][['last_update_time','feature']]
warning_data_df.sort_values(by='last_update_time',ascending=False,inplace=True)
warning_data['data']['WARNING_DATE'] = y_last_update_time
warning_data['data']['WARNING_CONTENT'] = warning_data_df.to_string(index=False)
token = get_head_auth_report()
upload_warning_data(token, warning_data)
logger.info(f'上传预警信息成功')
except Exception as e:
logger.error(f'上传预警信息失败:{e}')
# 去掉近最后数据对应的日期在六月以前的列删除近2月的数据是常熟的列
current_date = datetime.datetime.now()
two_months_ago = current_date - timedelta(days=180)
logger.info(f'删除两月不更新特征前数据量:{df.shape}')
def check_column(col_name):
'''
去掉空值列
@ -673,127 +699,6 @@ def getdata_juxiting(filename, datecol='date',y='y',dataset='',add_kdj=False,is_
return df
# def filter_data(ClassifyName,data):
# '''
# 指标名称保留规则
# '''
# # 包含 关键词 去除, 返回flase
# if any(keyword in data for keyword in ['运费','检修','波动率','地缘政治','股价',
# '同比','环比','环差','裂差','4WMA','变频','道琼斯','标普500','纳斯达克',
# '四周均值','名占比','残差','DMA',
# '连7-连9','4周平均','4周均值','滚动相关性','日本']):
# return False
# # 检查需要的特征
# # 去掉 分析 分类下的数据
# if ClassifyName == '分析':
# return False
# # 保留 库存中特殊关键词
# if ClassifyName == '库存':
# if any(keyword in data for keyword in ['原油' , '美国' ,'全球' ,'中国' ,'富查伊拉','ARA' ]):
# return True
# else:
# pass
# else:
# pass
# # 去掉 持仓中不是基金的数据
# if ClassifyName == '持仓':
# if '基金' not in data:
# return False
# else:
# pass
# else:
# pass
# # 去掉 航班中不是中国、美国 的数据
# if ClassifyName == '需求':
# if '航班' in data :
# if '中国' in data or '美国' in data :
# return True
# else:
# return False
# else:
# pass
# else:
# pass
# # 分类为 期货市场,同质性数据取第一个
# if ClassifyName == '期货市场':
# # 去掉c1-9 以后的
# if 'c1-c' in data:
# try:
# c = int(data.split('c1-c')[1])
# except:
# return False
# if c > 9 :
# return False
# else:
# pass
# else:
# pass
# # 判断 同质性数据, 字符串开头
# strstartdict = {'ICE Brent c':"ICE Brent c14",
# 'NYMWX WTI c':"NYMWX WTI c5",
# 'INE SC c':"INE SC c1",
# 'EFS c':"EFS c",
# 'Dubai Swap c':"Dubai Swap c1",
# 'Oman Swap c':"Oman Swap c1",
# 'DME Oman c':"DME Oman c1",
# 'Murban Futures c':"Murban Futures c1",
# 'Dubai连合约价格':'Dubai连1合约价格',
# '美国RBOB期货月份合约价格':'美国RBOB期货2309月份合约价格',
# 'Brent连合约价格':'Brent连1合约价格',
# 'WTI连合约价格':'WTI连1合约价格',
# '布伦特连合约价格':'Brent连1合约价格',
# 'Brent 连合约价格':'Brent连1合约价格',
# 'Dubai连合约价格':'Dubai连1合约价格',
# 'Brent连':'Brent连1合约价格',
# 'brent连':'Brent连1合约价格',
# }
# # 判断名称字符串开头是否在 strstartdict.keys中
# match = re.match(r'([a-zA-Z\s]+)(\d+)', data)
# if match:
# part1 = match.group(1)
# part2 = match.group(2)
# if part1 in [i for i in strstartdict.keys()]:
# if data == strstartdict[part1]:
# return True
# else:
# return False
# # data = 'Brent 连7合约价格'
# # 判断名称字符串去掉数字后是否在 strstartdict.keys中
# match = re.findall(r'\D+', data)
# if match :
# if len(match) == 2:
# part1 = match[0]
# part2 = match[1]
# if part1+part2 in [i for i in strstartdict.keys()]:
# if data == strstartdict[part1+part2]:
# return True
# else:
# return False
# else:
# pass
# elif len(match) == 1:
# match = re.findall(r'\D+', data)
# part1 = match[0]
# if part1 in [i for i in strstartdict.keys()]:
# if data == strstartdict[part1]:
# return True
# else:
# return False
# else:
# pass
# else:
# pass
# return True
def sanitize_filename(filename):
# 使用正则表达式替换不合规的字符

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,178 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"id": "31c0e11d-c87a-4e95-92a0-d1d09625e255",
"metadata": {},
"outputs": [],
"source": [
"from config_jingbo import *\n",
"import requests\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "83c81b9e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'http://192.168.100.53:8080/jingbo-dev/api/server/login'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"login_pushreport_url\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "2b330ee3-c006-4ab1-8558-59c51ac8d86f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'data': {'account': 'api_test',\n",
" 'password': 'ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=',\n",
" 'tenantHashCode': '8a4577dbd919675758d57999a1e891fe',\n",
" 'terminal': 'API'},\n",
" 'funcModule': 'API',\n",
" 'funcOperation': '获取token'}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"login_data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dcb6100a-ed2b-4077-a1a9-361c6cb565f9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzE5NDMzODgsImp0aSI6ImMwMTY4NWI1NTFlMzQyNGQ5M2JkZDIyYmE4YzI3YjY5In0.cj5Py63Hl0VML75Yy5VyFz4le-aM3VtFPXjjuwS8Y3w', 'md5Token': 'bd259af3a6d7e158204664387e18c4f5'}, 'status': True}\n"
]
}
],
"source": [
"\n",
"def get_head_auth_report():\n",
" login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 5))\n",
" text = json.loads(login_res.text)\n",
" print(text)\n",
" if text[\"status\"]:\n",
" token = text[\"data\"][\"accessToken\"]\n",
" return token\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "22c0c7c4",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'get_head_auth_report' is not defined",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[10], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m get_head_auth_report()\n",
"\u001b[1;31mNameError\u001b[0m: name 'get_head_auth_report' is not defined"
]
}
],
"source": [
"token = get_head_auth_report()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "12077ead",
"metadata": {},
"outputs": [],
"source": [
"token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzE5NDQ1MjMsImp0aSI6IjMxOTFlYTFmMDljOTQ3MjQ5OGIxYmEzMWZiYzUyNWE5In0.XEvtFDbN6Qd9ibTEzLWykcDSgiQQOgBro6vTzXiwbgQ'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a7ae21d1",
"metadata": {},
"outputs": [],
"source": [
"warning_data = {'funcModule': '原油特征停更预警', \n",
" 'funcOperation': '原油特征停更预警', \n",
" 'data': {'WARNING_TYPE_NAME': '日度数据', \n",
" 'WARNING_CONTENT': ''' feature last_update_time update_period warning_date\\n 美国:东海岸地区:炼油厂的投入与使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n 美国:炼油厂的投入与 使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n 美国:洛基山地区:炼油厂的投入与使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n \n",
" 美国:墨西哥湾沿岸:炼油厂的投入与使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n 美国:西海岸地区:炼油厂的投入与使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n 美国:中西部地区:炼油厂的投入与使用情况:开工率:当周值 2024-06-14 7.0 2024-06-29\\n 中国航班执行数 2024-04-10 1.0 2024-04-13\\n 美国汽油产量占比 2024-10-04 7.0 2024-10-19\\n 美国柴油产量占比 2024-10-04 7.0 2024-10-19\\n 西北欧即期Brent 211利润 2018-10-24 1.0 2018-10-27\\n 美国RBOB期货2309月份合约价格 2023-08-31 1.0 2023-09-03\\nPVM Dubai EFS/Swap vs ICE First Month Brent 2023-01-03 1.0 2023-01-06\\n Ebob连1合约 2024-02-28 1.0 2024-03-02\\n Brent/迪拜原油升贴水 2022-11-16 1.0 2022-11-19\\n W-B活跃合约价差 2022-01-31 1.0 2022-02-03\\n B-W活跃合约价差 2022-01-31 1.0 2022-02-03\\n Brent c2-c3 (结算B) 2024-10-31 1.0 2024-11-03\\n WTI c2-c3 (结算B) 2024-10-31 1.0 2024-11-03\\n Kpler-海运在途原油 2024-08-20 1.0 2024-08-23\\n 全球原油库存(剔除富查伊拉) 2024-07-21 7.0 2024-08-05\\n 原油:山东港口:库存:新口径(周)超季节性/3年 2024-05-24 7.0 2024-06-08\\n 原油:港口库存:中国(周) 2023-08-04 7.0 2023-08-19\\n 原油:山东港口:库存:新口径(周) 2024-05-24 7.0 2024-06-08\\n 美国墨西哥湾87#汽油现货价格 2024-01-31 1.0 2024-02-03\\n 欧洲汽油裂解 2024-11-01 1.0 2024-11-04\\n Brent c1 02:30收盘价滞后1天 2024-10-30 1.0 2024-11-02\\n Brent c2 02:30收盘价滞后1天 2024-10-30 1.0 2024-11-02\\n EFS c1 02:30收盘价滞后1天 2024-10-30 1.0 2024-11-02\\n EFS c2 02:30收盘价滞后1天 2024-10-30 1.0 2024-11-02\\n 原油:开工率:东北地区:独立炼厂(周) 2023-03-31 7.0 2023-04-15\\n 美国墨西哥湾原油钻机数 2024-09-27 7.0 2024-10-12\\n Kpler-美国原油净进口(周度) 2024-08-25 7.0 2024-09-09\\n 中东-也门周度原油海运出口 2023-08-27 21.0 2023-10-09\\n 中东-巴林周度原油海运出口 2024-04-28 7.0 2024-05-13' ''', \n",
" 'WARNING_DATE': '2024-11-14'}}"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3e1a9e67",
"metadata": {},
"outputs": [],
"source": [
"# def upload_warning_data(token, warning_data):\n",
"warning_data = warning_data\n",
"headers = {\"Authorization\": token}\n",
"logger.info(\"预警上传中...\")\n",
"logger.info(f\"token:{token}\")\n",
"logger.info(f\"warning_data:{warning_data}\" )\n",
"upload_res = requests.post(url=upload_warning_url, headers=headers, json=warning_data, timeout=(3, 15))\n",
"print(upload_res)\n",
" # if upload_res:\n",
" # return upload_res\n",
" # else:\n",
" # logger.info(\"预警上传失败\")\n",
" # return None"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}